Java Code Examples for org.apache.hadoop.fs.FSDataInputStream#getPos()

The following examples show how to use org.apache.hadoop.fs.FSDataInputStream#getPos() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Tail.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private long dumpFromOffset(PathData item, long offset) throws IOException {
  long fileSize = item.refreshStatus().getLen();
  if (offset > fileSize) return fileSize;
  // treat a negative offset as relative to end of the file, floor of 0
  if (offset < 0) {
    offset = Math.max(fileSize + offset, 0);
  }
  
  FSDataInputStream in = item.fs.open(item.path);
  try {
    in.seek(offset);
    // use conf so the system configured io block size is used
    IOUtils.copyBytes(in, System.out, getConf(), false);
    offset = in.getPos();
  } finally {
    in.close();
  }
  return offset;
}
 
Example 2
Source File: MRInputUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk(
    TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter)
    throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  long offset = splitMetaInfo.getStartOffset();

  // Split information read from local filesystem.
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapreduce.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapreduce.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapreduce.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapreduce.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapreduce.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
 
Example 3
Source File: MapTask.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> T getSplitDetails(Path file, long offset) 
 throws IOException {
  FileSystem fs = file.getFileSystem(conf);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = StringInterner.weakIntern(Text.readString(inFile));
  Class<T> cls;
  try {
    cls = (Class<T>) conf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + 
                                        " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(conf);
  Deserializer<T> deserializer = 
    (Deserializer<T>) factory.getDeserializer(cls);
  deserializer.open(inFile);
  T split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  getCounters().findCounter(
      TaskCounter.SPLIT_RAW_BYTES).increment(pos - offset);
  inFile.close();
  return split;
}
 
Example 4
Source File: MRInputUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
 
Example 5
Source File: MapTask.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private <T> T getSplitDetails(Path file, long offset) 
 throws IOException {
  FileSystem fs = file.getFileSystem(conf);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = StringInterner.weakIntern(Text.readString(inFile));
  Class<T> cls;
  try {
    cls = (Class<T>) conf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + 
                                        " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(conf);
  Deserializer<T> deserializer = 
    (Deserializer<T>) factory.getDeserializer(cls);
  deserializer.open(inFile);
  T split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  getCounters().findCounter(
      TaskCounter.SPLIT_RAW_BYTES).increment(pos - offset);
  inFile.close();
  return split;
}
 
Example 6
Source File: MRInputUtils.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static org.apache.hadoop.mapreduce.InputSplit getNewSplitDetailsFromDisk(
    TaskSplitIndex splitMetaInfo, JobConf jobConf, TezCounter splitBytesCounter)
    throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  long offset = splitMetaInfo.getStartOffset();

  // Split information read from local filesystem.
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapreduce.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapreduce.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapreduce.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapreduce.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapreduce.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
 
Example 7
Source File: MRInputUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public static InputSplit getOldSplitDetailsFromDisk(TaskSplitIndex splitMetaInfo,
    JobConf jobConf, TezCounter splitBytesCounter) throws IOException {
  Path file = new Path(splitMetaInfo.getSplitLocation());
  FileSystem fs = FileSystem.getLocal(jobConf);
  file = fs.makeQualified(file);
  LOG.info("Reading input split file from : " + file);
  long offset = splitMetaInfo.getStartOffset();

  FSDataInputStream inFile = fs.open(file);
  inFile.seek(offset);
  String className = Text.readString(inFile);
  Class<org.apache.hadoop.mapred.InputSplit> cls;
  try {
    cls = (Class<org.apache.hadoop.mapred.InputSplit>) jobConf.getClassByName(className);
  } catch (ClassNotFoundException ce) {
    IOException wrap = new IOException("Split class " + className + " not found");
    wrap.initCause(ce);
    throw wrap;
  }
  SerializationFactory factory = new SerializationFactory(jobConf);
  Deserializer<org.apache.hadoop.mapred.InputSplit> deserializer = (Deserializer<org.apache.hadoop.mapred.InputSplit>) factory
      .getDeserializer(cls);
  deserializer.open(inFile);
  org.apache.hadoop.mapred.InputSplit split = deserializer.deserialize(null);
  long pos = inFile.getPos();
  if (splitBytesCounter != null) {
    splitBytesCounter.increment(pos - offset);
  }
  inFile.close();
  return split;
}
 
Example 8
Source File: CBZip2InputStream.java    From spork with Apache License 2.0 5 votes vote down vote up
public CBZip2InputStream(FSDataInputStream zStream, int blockSize, long end)
throws IOException {
    endOffsetOfSplit = end;
    // initialize retPos to the beginning of the current InputSplit
    // see comments in getPos() to understand how this is used.
    retPos = zStream.getPos();
	ll8 = null;
    tt = null;
    checkComputedCombinedCRC = blockSize == -1;
    bsSetStream(zStream);
    initialize(blockSize);
    initBlock(blockSize != -1);
    setupBlock();
}
 
Example 9
Source File: HdfsKeyValueStore.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private void loadIndex(Path path) throws IOException {
  FSDataInputStream inputStream = _fileSystem.open(path);
  byte[] buf = new byte[MAGIC.length];
  inputStream.readFully(buf);
  if (!Arrays.equals(MAGIC, buf)) {
    throw new IOException("File [" + path + "] not a " + BLUR_KEY_VALUE + " file.");
  }
  int version = inputStream.readInt();
  if (version == 1) {
    long fileLength = HdfsUtils.getFileLength(_fileSystem, path, inputStream);
    Operation operation = new Operation();
    try {
      while (inputStream.getPos() < fileLength) {
        try {
          operation.readFields(inputStream);
        } catch (IOException e) {
          // End of sync point found
          return;
        }
        loadIndex(path, operation);
      }
    } finally {
      inputStream.close();
    }
  } else {
    throw new IOException("Unknown version [" + version + "]");
  }
}
 
Example 10
Source File: ProtobufLogReader.java    From hbase with Apache License 2.0 5 votes vote down vote up
private String initInternal(FSDataInputStream stream, boolean isFirst)
    throws IOException {
  close();
  long expectedPos = PB_WAL_MAGIC.length;
  if (stream == null) {
    stream = fs.open(path);
    stream.seek(expectedPos);
  }
  if (stream.getPos() != expectedPos) {
    throw new IOException("The stream is at invalid position: " + stream.getPos());
  }
  // Initialize metadata or, when we reset, just skip the header.
  WALProtos.WALHeader.Builder builder = WALProtos.WALHeader.newBuilder();
  WALHdrContext hdrCtxt = readHeader(builder, stream);
  WALHdrResult walHdrRes = hdrCtxt.getResult();
  if (walHdrRes == WALHdrResult.EOF) {
    throw new EOFException("Couldn't read WAL PB header");
  }
  if (walHdrRes == WALHdrResult.UNKNOWN_WRITER_CLS) {
    throw new IOException("Got unknown writer class: " + builder.getWriterClsName());
  }
  if (isFirst) {
    WALProtos.WALHeader header = builder.build();
    this.hasCompression = header.hasHasCompression() && header.getHasCompression();
    this.hasTagCompression = header.hasHasTagCompression() && header.getHasTagCompression();
  }
  this.inputStream = stream;
  this.walEditsStopOffset = this.fileLength;
  long currentPosition = stream.getPos();
  trailerPresent = setTrailerIfPresent();
  this.seekOnFs(currentPosition);
  if (LOG.isTraceEnabled()) {
    LOG.trace("After reading the trailer: walEditsStopOffset: " + this.walEditsStopOffset
        + ", fileLength: " + this.fileLength + ", " + "trailerPresent: " + (trailerPresent ? "true, size: " + trailer.getSerializedSize() : "false") + ", currentPosition: " + currentPosition);
  }
  
  codecClsName = hdrCtxt.getCellCodecClsName();
  
  return hdrCtxt.getCellCodecClsName();
}
 
Example 11
Source File: SuccinctIndexedFileStream.java    From succinct with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor to map a file containing Succinct data structures via stream.
 *
 * @param filePath Path of the file.
 * @param conf     Configuration for the filesystem.
 * @throws IOException
 */
public SuccinctIndexedFileStream(Path filePath, Configuration conf) throws IOException {
  super(filePath, conf);
  FSDataInputStream is = getStream(filePath);
  is.seek(endOfFileStream);
  int len = is.readInt();
  offsets = new int[len];
  for (int i = 0; i < len; i++) {
    offsets[i] = is.readInt();
  }
  endOfIndexedFileStream = is.getPos();
  is.close();
}
 
Example 12
Source File: FSDataInputChannel.java    From tajo with Apache License 2.0 5 votes vote down vote up
public FSDataInputChannel(FSDataInputStream inputStream) throws IOException {
  if (inputStream.getWrappedStream() instanceof ByteBufferReadable) {
    this.isDirectRead = true;
  } else {
    /* LocalFileSystem, S3 does not support ByteBufferReadable */
    this.channel = Channels.newChannel(inputStream);
  }
  this.inputStream = inputStream;
  this.size = inputStream.getPos() + inputStream.available();
}
 
Example 13
Source File: AbstractFileOutputOperator.java    From attic-apex-malhar with Apache License 2.0 4 votes vote down vote up
/**
 * Recovers a file which exists on the disk. If the length of the file is not same as the
 * length which the operator remembers then the file is truncated. <br/>
 * When always writing to a temporary file, then a file is restored even when the length is same as what the
 * operator remembers however this is done only for files which had open streams that weren't closed before
 * failure.
 *
 * @param filename     name of the actual file.
 * @param partFileName name of the part file. When not rolling this is same as filename; otherwise this is the
 *                     latest open part file name.
 * @param filepath     path of the file. When always writing to temp file, this is the path of the temp file;
 *                     otherwise path of the actual file.
 * @throws IOException
 */
private void recoverFile(String filename, String partFileName, Path filepath) throws IOException
{
  LOG.debug("path exists {}", filepath);
  long offset = endOffsets.get(filename).longValue();
  FSDataInputStream inputStream = fs.open(filepath);
  FileStatus status = fs.getFileStatus(filepath);

  if (status.getLen() != offset) {
    LOG.info("path corrupted {} {} {}", filepath, offset, status.getLen());
    byte[] buffer = new byte[COPY_BUFFER_SIZE];
    String recoveryFileName = partFileName + '.' + System.currentTimeMillis() + TMP_EXTENSION;
    Path recoveryFilePath = new Path(filePath + Path.SEPARATOR + recoveryFileName);
    FSDataOutputStream fsOutput = openStream(recoveryFilePath, false);

    while (inputStream.getPos() < offset) {
      long remainingBytes = offset - inputStream.getPos();
      int bytesToWrite = remainingBytes < COPY_BUFFER_SIZE ? (int)remainingBytes : COPY_BUFFER_SIZE;
      inputStream.read(buffer);
      fsOutput.write(buffer, 0, bytesToWrite);
    }

    flush(fsOutput);
    fsOutput.close();
    inputStream.close();

    LOG.debug("active {} recovery {} ", filepath, recoveryFilePath);

    if (alwaysWriteToTmp) {
      //recovery file is used as the new tmp file and we cannot delete the old tmp file because when the operator
      //is restored to an earlier check-pointed window, it will look for an older tmp.
      fileNameToTmpName.put(partFileName, recoveryFileName);
    } else {
      LOG.debug("recovery path {} actual path {} ", recoveryFilePath, status.getPath());
      rename(recoveryFilePath, status.getPath());
    }
  } else {
    if (alwaysWriteToTmp && filesWithOpenStreams.contains(filename)) {
      String currentTmp = partFileName + '.' + System.currentTimeMillis() + TMP_EXTENSION;
      FSDataOutputStream outputStream = openStream(new Path(filePath + Path.SEPARATOR + currentTmp), false);
      IOUtils.copy(inputStream, outputStream);
      streamsCache.put(filename, new FSFilterStreamContext(outputStream));
      fileNameToTmpName.put(partFileName, currentTmp);
    }
    inputStream.close();
  }
}
 
Example 14
Source File: DataValidationInputFormat.java    From jumbune with GNU Lesser General Public License v3.0 4 votes vote down vote up
/**
 * Generate splits.
 *
 * @param job refers to JobContext that is being used to read the configurations of the job that ran
 * @param minSize refers to the minimum file block size.
 * @param maxSize refers to the maximum file block size.
 * @param splits refers  to a list of splits that are being generated.
 * @param file refers to the FileStatus required to determine block size,length,allocations.
 * @throws IOException Signals that an I/O exception has occurred.
 */
private void generateSplits(JobContext job, long minSize, long maxSize,
		List<InputSplit> splits, FileStatus file) throws IOException {
	Path path = file.getPath();
	int numOfRecordsInCurrentSplit = 0;
	int numOfRecordsInPreviousSplit = 0;
	FileSystem fs = path.getFileSystem(job.getConfiguration());
	long length = file.getLen();
	BlockLocation[] blkLocations = fs.getFileBlockLocations(file, 0,
			length);
	FSDataInputStream fsin = null ;
	if ((length != 0) && isSplitable(job, path)) {
		long blockSize = file.getBlockSize();
		long splitSize = computeSplitSize(blockSize, minSize, maxSize);
		long bytesRemaining = length;
		
		// checking the occurrences of the record separator in current
		// split
		recordSeparator = job.getConfiguration()
				.get(DataValidationConstants.RECORD_SEPARATOR)
				.getBytes();
		while (((double) bytesRemaining) / splitSize > SPLIT_SLOP) {
			int blkIndex = getBlockIndex(blkLocations, length
					- bytesRemaining);
			long start = length - bytesRemaining;
			long end = start + splitSize;
			try{
			fsin = fs.open(path);
			fsin.seek(start);
			long pos = start;
			int b = 0;
			int bufferPos = 0;
			while (true) {
				b = fsin.read();
				pos = fsin.getPos();
				if (b == -1) {
					break;}
				if (b == recordSeparator[bufferPos]) {
					bufferPos++;
					if (bufferPos == recordSeparator.length) {
						numOfRecordsInCurrentSplit++;
						bufferPos = 0;
						if (pos > end) {
							break;
						}
					}
				} else {
					// reset the value of buffer position to zero
					bufferPos = 0;
				}

			}}finally{
				if(fsin != null){
					fsin.close();
				}
			}

			splits.add(new DataValidationFileSplit(path, start,
					splitSize, numOfRecordsInPreviousSplit,
					blkLocations[blkIndex].getHosts()));
			bytesRemaining -= splitSize;
			numOfRecordsInPreviousSplit = numOfRecordsInCurrentSplit;
			numOfRecordsInCurrentSplit = 0;
		}

		addSplitIfBytesRemaining(splits, path, numOfRecordsInPreviousSplit,
				length, blkLocations, bytesRemaining);
	} else if (length != 0) {
		splits.add(new DataValidationFileSplit(path, 0, length,
				numOfRecordsInPreviousSplit, blkLocations[0].getHosts()));
	} else {
		splits.add(new DataValidationFileSplit(path, 0, length,
				numOfRecordsInPreviousSplit, new String[0]));
	}
}
 
Example 15
Source File: HdfsIOBenchmark.java    From incubator-crail with Apache License 2.0 4 votes vote down vote up
public void readSequentialDirect() throws Exception {
	System.out.println("reading sequential file in direct mode " + path);
	Configuration conf = new Configuration();
	FileSystem fs = FileSystem.get(conf);
	FileStatus status = fs.getFileStatus(path);
	FSDataInputStream instream = fs.open(path);
	ByteBuffer buf = ByteBuffer.allocateDirect(size);
	buf.clear();
	double sumbytes = 0;
	double ops = 0;
	System.out.println("file capacity " + status.getLen());
	System.out.println("read size " + size);
	System.out.println("operations " + loop);
	
	long start = System.currentTimeMillis();
	while (ops < loop) {
		buf.clear();
		double ret = (double) instream.read(buf);
		if (ret > 0) {
			sumbytes = sumbytes + ret;
			ops = ops + 1.0;
		} else {
			ops = ops + 1.0;
			if (instream.getPos() == 0){
				break;
			} else {
				instream.seek(0);
			}
		}
	}
	long end = System.currentTimeMillis();
	double executionTime = ((double) (end - start)) / 1000.0;
	double throughput = 0.0;
	double latency = 0.0;
	double sumbits = sumbytes * 8.0;
	if (executionTime > 0) {
		throughput = sumbits / executionTime / 1024.0 / 1024.0;
		latency = 1000000.0 * executionTime / ops;
	}
	System.out.println("execution time " + executionTime);
	System.out.println("ops " + ops);
	System.out.println("sumbytes " + sumbytes);
	System.out.println("throughput " + throughput);
	System.out.println("latency " + latency);
	System.out.println("closing stream");
	instream.close();	
	fs.close();
}
 
Example 16
Source File: HdfsIOBenchmark.java    From crail with Apache License 2.0 4 votes vote down vote up
public void readSequentialHeap() throws Exception {
	System.out.println("reading sequential file in heap mode " + path);
	Configuration conf = new Configuration();
	FileSystem fs = FileSystem.get(conf);
	FileStatus status = fs.getFileStatus(path);
	FSDataInputStream instream = fs.open(path);
	byte[] buf = new byte[size];
	double sumbytes = 0;
	double ops = 0;
	System.out.println("file capacity " + status.getLen());
	System.out.println("read size " + size);
	System.out.println("operations " + loop);
	
	long start = System.currentTimeMillis();
	while (ops < loop) {
		double ret = (double) this.read(instream, buf);
		if (ret > 0) {
			sumbytes = sumbytes + ret;
			ops = ops + 1.0;
		} else {
			ops = ops + 1.0;
			if (instream.getPos() == 0){
				break;
			} else {
				instream.seek(0);
			}
		}
	}
	long end = System.currentTimeMillis();
	double executionTime = ((double) (end - start)) / 1000.0;
	double throughput = 0.0;
	double latency = 0.0;
	double sumbits = sumbytes * 8.0;
	if (executionTime > 0) {
		throughput = sumbits / executionTime / 1024.0 / 1024.0;
		latency = 1000000.0 * executionTime / ops;
	}
	System.out.println("execution time " + executionTime);
	System.out.println("ops " + ops);
	System.out.println("sumbytes " + sumbytes);
	System.out.println("throughput " + throughput);
	System.out.println("latency " + latency);
	System.out.println("closing stream");
	instream.close();	
	fs.close();
}
 
Example 17
Source File: HdfsIOBenchmark.java    From crail with Apache License 2.0 4 votes vote down vote up
public void readSequentialDirect() throws Exception {
	System.out.println("reading sequential file in direct mode " + path);
	Configuration conf = new Configuration();
	FileSystem fs = FileSystem.get(conf);
	FileStatus status = fs.getFileStatus(path);
	FSDataInputStream instream = fs.open(path);
	ByteBuffer buf = ByteBuffer.allocateDirect(size);
	buf.clear();
	double sumbytes = 0;
	double ops = 0;
	System.out.println("file capacity " + status.getLen());
	System.out.println("read size " + size);
	System.out.println("operations " + loop);
	
	long start = System.currentTimeMillis();
	while (ops < loop) {
		buf.clear();
		double ret = (double) instream.read(buf);
		if (ret > 0) {
			sumbytes = sumbytes + ret;
			ops = ops + 1.0;
		} else {
			ops = ops + 1.0;
			if (instream.getPos() == 0){
				break;
			} else {
				instream.seek(0);
			}
		}
	}
	long end = System.currentTimeMillis();
	double executionTime = ((double) (end - start)) / 1000.0;
	double throughput = 0.0;
	double latency = 0.0;
	double sumbits = sumbytes * 8.0;
	if (executionTime > 0) {
		throughput = sumbits / executionTime / 1024.0 / 1024.0;
		latency = 1000000.0 * executionTime / ops;
	}
	System.out.println("execution time " + executionTime);
	System.out.println("ops " + ops);
	System.out.println("sumbytes " + sumbytes);
	System.out.println("throughput " + throughput);
	System.out.println("latency " + latency);
	System.out.println("closing stream");
	instream.close();	
	fs.close();
}
 
Example 18
Source File: TestWriteRead.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * read chunks into buffer repeatedly until total of VisibleLen byte are read.
 * Return total number of bytes read
 */
private long readUntilEnd(FSDataInputStream in, byte[] buffer, long size,
    String fname, long pos, long visibleLen, boolean positionReadOption)
    throws IOException {

  if (pos >= visibleLen || visibleLen <= 0)
    return 0;

  int chunkNumber = 0;
  long totalByteRead = 0;
  long currentPosition = pos;
  int byteRead = 0;
  long byteLeftToRead = visibleLen - pos;
  int byteToReadThisRound = 0;

  if (!positionReadOption) {
    in.seek(pos);
    currentPosition = in.getPos();
  }
  if (verboseOption)
    LOG.info("reader begin: position: " + pos + " ; currentOffset = "
        + currentPosition + " ; bufferSize =" + buffer.length
        + " ; Filename = " + fname);
  try {
    while (byteLeftToRead > 0 && currentPosition < visibleLen) {
      byteToReadThisRound = (int) (byteLeftToRead >= buffer.length 
          ? buffer.length : byteLeftToRead);
      if (positionReadOption) {
        byteRead = in.read(currentPosition, buffer, 0, byteToReadThisRound);
      } else {
        byteRead = in.read(buffer, 0, byteToReadThisRound);
      }
      if (byteRead <= 0)
        break;
      chunkNumber++;
      totalByteRead += byteRead;
      currentPosition += byteRead;
      byteLeftToRead -= byteRead;

      if (verboseOption) {
        LOG.info("reader: Number of byte read: " + byteRead
            + " ; totalByteRead = " + totalByteRead + " ; currentPosition="
            + currentPosition + " ; chunkNumber =" + chunkNumber
            + "; File name = " + fname);
      }
    }
  } catch (IOException e) {
    throw new IOException(
        "#### Exception caught in readUntilEnd: reader  currentOffset = "
            + currentPosition + " ; totalByteRead =" + totalByteRead
            + " ; latest byteRead = " + byteRead + "; visibleLen= "
            + visibleLen + " ; bufferLen = " + buffer.length
            + " ; Filename = " + fname, e);
  }

  if (verboseOption)
    LOG.info("reader end:   position: " + pos + " ; currentOffset = "
        + currentPosition + " ; totalByteRead =" + totalByteRead
        + " ; Filename = " + fname);

  return totalByteRead;
}
 
Example 19
Source File: TestWriteRead.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * read chunks into buffer repeatedly until total of VisibleLen byte are read.
 * Return total number of bytes read
 */
private long readUntilEnd(FSDataInputStream in, byte[] buffer, long size,
    String fname, long pos, long visibleLen, boolean positionReadOption)
    throws IOException {

  if (pos >= visibleLen || visibleLen <= 0)
    return 0;

  int chunkNumber = 0;
  long totalByteRead = 0;
  long currentPosition = pos;
  int byteRead = 0;
  long byteLeftToRead = visibleLen - pos;
  int byteToReadThisRound = 0;

  if (!positionReadOption) {
    in.seek(pos);
    currentPosition = in.getPos();
  }
  if (verboseOption)
    LOG.info("reader begin: position: " + pos + " ; currentOffset = "
        + currentPosition + " ; bufferSize =" + buffer.length
        + " ; Filename = " + fname);
  try {
    while (byteLeftToRead > 0 && currentPosition < visibleLen) {
      byteToReadThisRound = (int) (byteLeftToRead >= buffer.length 
          ? buffer.length : byteLeftToRead);
      if (positionReadOption) {
        byteRead = in.read(currentPosition, buffer, 0, byteToReadThisRound);
      } else {
        byteRead = in.read(buffer, 0, byteToReadThisRound);
      }
      if (byteRead <= 0)
        break;
      chunkNumber++;
      totalByteRead += byteRead;
      currentPosition += byteRead;
      byteLeftToRead -= byteRead;

      if (verboseOption) {
        LOG.info("reader: Number of byte read: " + byteRead
            + " ; totalByteRead = " + totalByteRead + " ; currentPosition="
            + currentPosition + " ; chunkNumber =" + chunkNumber
            + "; File name = " + fname);
      }
    }
  } catch (IOException e) {
    throw new IOException(
        "#### Exception caught in readUntilEnd: reader  currentOffset = "
            + currentPosition + " ; totalByteRead =" + totalByteRead
            + " ; latest byteRead = " + byteRead + "; visibleLen= "
            + visibleLen + " ; bufferLen = " + buffer.length
            + " ; Filename = " + fname, e);
  }

  if (verboseOption)
    LOG.info("reader end:   position: " + pos + " ; currentOffset = "
        + currentPosition + " ; totalByteRead =" + totalByteRead
        + " ; Filename = " + fname);

  return totalByteRead;
}
 
Example 20
Source File: HdfsIOBenchmark.java    From incubator-crail with Apache License 2.0 4 votes vote down vote up
public void readSequentialHeap() throws Exception {
	System.out.println("reading sequential file in heap mode " + path);
	Configuration conf = new Configuration();
	FileSystem fs = FileSystem.get(conf);
	FileStatus status = fs.getFileStatus(path);
	FSDataInputStream instream = fs.open(path);
	byte[] buf = new byte[size];
	double sumbytes = 0;
	double ops = 0;
	System.out.println("file capacity " + status.getLen());
	System.out.println("read size " + size);
	System.out.println("operations " + loop);
	
	long start = System.currentTimeMillis();
	while (ops < loop) {
		double ret = (double) this.read(instream, buf);
		if (ret > 0) {
			sumbytes = sumbytes + ret;
			ops = ops + 1.0;
		} else {
			ops = ops + 1.0;
			if (instream.getPos() == 0){
				break;
			} else {
				instream.seek(0);
			}
		}
	}
	long end = System.currentTimeMillis();
	double executionTime = ((double) (end - start)) / 1000.0;
	double throughput = 0.0;
	double latency = 0.0;
	double sumbits = sumbytes * 8.0;
	if (executionTime > 0) {
		throughput = sumbits / executionTime / 1024.0 / 1024.0;
		latency = 1000000.0 * executionTime / ops;
	}
	System.out.println("execution time " + executionTime);
	System.out.println("ops " + ops);
	System.out.println("sumbytes " + sumbytes);
	System.out.println("throughput " + throughput);
	System.out.println("latency " + latency);
	System.out.println("closing stream");
	instream.close();	
	fs.close();
}