Java Code Examples for org.apache.hadoop.io.WritableUtils#readVLong()

/**
 * Reads a bitset from the stream that has been written with
 * {@link #writeBitSet(DataOutput, int, BitSet)}.
 */
private static final void readBitSet(DataInput stream, int nbits, 
    BitSet bitSet) throws IOException {
  bitSet.clear();
  long initialBits = WritableUtils.readVLong(stream);
  long last = 0L;
  while (0L != initialBits) {
    last = Long.lowestOneBit(initialBits);
    initialBits ^= last;
    bitSet.set(Long.numberOfTrailingZeros(last));
  }
  
  for (int offset=Long.SIZE; offset < nbits; offset+=Byte.SIZE) {
    byte bits = stream.readByte();
    while (0 != bits) {
      last = Long.lowestOneBit(bits);
      bits ^= last;
      bitSet.set(Long.numberOfTrailingZeros(last) + offset);
    }
  }
}
 

/** This constructor is there primarily to serve the sort routine that 
 * generates a single output file with an associated index file */
public SegmentContainer(Path inName, Path indexIn) throws IOException {
  //get the segments from indexIn
  FSDataInputStream fsIndexIn = fs.open(indexIn);
  long end = fs.getFileStatus(indexIn).getLen();
  while (fsIndexIn.getPos() < end) {
    long segmentOffset = WritableUtils.readVLong(fsIndexIn);
    long segmentLength = WritableUtils.readVLong(fsIndexIn);
    Path segmentName = inName;
    segments.add(new LinkedSegmentsDescriptor(segmentOffset, 
                                              segmentLength, segmentName, this));
  }
  fsIndexIn.close();
  fs.delete(indexIn, true);
  numSegmentsContained = segments.size();
  this.inName = inName;
}
 

@Override
public void readFields(DataInput in) throws IOException {
  id = WritableUtils.readVInt(in);
  sleepDuration = WritableUtils.readVLong(in);
  nMaps = WritableUtils.readVInt(in);
  nSpec = WritableUtils.readVInt(in);
  if (reduceDurations.length < nSpec) {
    reduceDurations = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceDurations[i] = WritableUtils.readVLong(in);
  }
  final int nLoc = WritableUtils.readVInt(in);
  if (nLoc != locations.length) {
    locations = new String[nLoc];
  }
  for (int i = 0; i < nLoc; ++i) {
    locations[i] = Text.readString(in);
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  super.readFields(in);
  id = WritableUtils.readVInt(in);
  maps = WritableUtils.readVInt(in);
  inputRecords = WritableUtils.readVLong(in);
  outputBytes = WritableUtils.readVLong(in);
  outputRecords = WritableUtils.readVLong(in);
  maxMemory = WritableUtils.readVLong(in);
  reduces = WritableUtils.readVInt(in);
  if (reduceBytes.length < reduces) {
    reduceBytes = new double[reduces];
    reduceRecords = new double[reduces];
  }
  for (int i = 0; i < reduces; ++i) {
    reduceBytes[i] = in.readDouble();
    reduceRecords[i] = in.readDouble();
  }
  nSpec = WritableUtils.readVInt(in);
  if (reduceOutputBytes.length < nSpec) {
    reduceOutputBytes = new long[nSpec];
    reduceOutputRecords = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceOutputBytes[i] = WritableUtils.readVLong(in);
    reduceOutputRecords[i] = WritableUtils.readVLong(in);
  }
  mapMetrics = new ResourceUsageMetrics();
  mapMetrics.readFields(in);
  int numReduceMetrics = WritableUtils.readVInt(in);
  reduceMetrics = new ResourceUsageMetrics[numReduceMetrics];
  for (int i = 0; i < numReduceMetrics; ++i) {
    reduceMetrics[i] = new ResourceUsageMetrics();
    reduceMetrics[i].readFields(in);
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  rec_in = WritableUtils.readVLong(in);
  rec_out = WritableUtils.readVLong(in);
  bytes_out = WritableUtils.readVLong(in);
  sizeOfResourceUsageMetrics =  WritableUtils.readVInt(in);
  if (sizeOfResourceUsageMetrics > 0) {
    metrics = new ResourceUsageMetrics();
    metrics.readFields(in);
  }
}
 

@Override
public void readFields(DataInput input) throws IOException {
    regionLocation = WritableUtils.readString(input);
    regionSize = WritableUtils.readVLong(input);
    int count = WritableUtils.readVInt(input);
    scans = Lists.newArrayListWithExpectedSize(count);
    for (int i = 0; i < count; i++) {
        byte[] protoScanBytes = new byte[WritableUtils.readVInt(input)];
        input.readFully(protoScanBytes);
        ClientProtos.Scan protoScan = ClientProtos.Scan.parseFrom(protoScanBytes);
        Scan scan = ProtobufUtil.toScan(protoScan);
        scans.add(scan);
    }
    init();
}
 

@Override
public void readFields(DataInput in) throws IOException {
  rec_in = WritableUtils.readVLong(in);
  rec_out = WritableUtils.readVLong(in);
  bytes_out = WritableUtils.readVLong(in);
  sizeOfResourceUsageMetrics =  WritableUtils.readVInt(in);
  if (sizeOfResourceUsageMetrics > 0) {
    metrics = new ResourceUsageMetrics();
    metrics.readFields(in);
  }
}
 

@Override
public synchronized void readFields(DataInput in) throws IOException {
  name = StringInterner.weakIntern(Text.readString(in));
  displayName = in.readBoolean() ? 
      StringInterner.weakIntern(Text.readString(in)) : name;
  value = WritableUtils.readVLong(in);
}
 

public void readFields(DataInput in) throws IOException {
  long recordTypeId = WritableUtils.readVLong(in);
  if (recordTypeId != INDEX_TABLE_ID) {
    // We expected to read an IndexTable.
    throw new IOException("Expected IndexTable; got record with typeId="
        + recordTypeId);
  }

  int tableCount = WritableUtils.readVInt(in);

  tableEntries = new ArrayList<IndexTableEntry>(tableCount);
  for (int i = 0; i < tableCount; i++) {
    tableEntries.add(new IndexTableEntry(in));
  }
}
 

/**
 * check if a token is expired. for unit test only. return true when token is
 * expired, false otherwise
 */
static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
    throws IOException {
  ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
  DataInputStream in = new DataInputStream(buf);
  long expiryDate = WritableUtils.readVLong(in);
  return isExpired(expiryDate);
}
 

@Override
public void readFields(DataInput in) throws IOException {
  super.readFields(in);
  id = WritableUtils.readVInt(in);
  maps = WritableUtils.readVInt(in);
  inputRecords = WritableUtils.readVLong(in);
  outputBytes = WritableUtils.readVLong(in);
  outputRecords = WritableUtils.readVLong(in);
  maxMemory = WritableUtils.readVLong(in);
  reduces = WritableUtils.readVInt(in);
  if (reduceBytes.length < reduces) {
    reduceBytes = new double[reduces];
    reduceRecords = new double[reduces];
  }
  for (int i = 0; i < reduces; ++i) {
    reduceBytes[i] = in.readDouble();
    reduceRecords[i] = in.readDouble();
  }
  nSpec = WritableUtils.readVInt(in);
  if (reduceOutputBytes.length < nSpec) {
    reduceOutputBytes = new long[nSpec];
    reduceOutputRecords = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceOutputBytes[i] = WritableUtils.readVLong(in);
    reduceOutputRecords[i] = WritableUtils.readVLong(in);
  }
  mapMetrics = new ResourceUsageMetrics();
  mapMetrics.readFields(in);
  int numReduceMetrics = WritableUtils.readVInt(in);
  reduceMetrics = new ResourceUsageMetrics[numReduceMetrics];
  for (int i = 0; i < numReduceMetrics; ++i) {
    reduceMetrics[i] = new ResourceUsageMetrics();
    reduceMetrics[i].readFields(in);
  }
}
 

@Override
public void readFields(DataInput in) throws IOException {
  this.cache = null;
  expiryDate = WritableUtils.readVLong(in);
  keyId = WritableUtils.readVInt(in);
  userId = WritableUtils.readString(in);
  blockPoolId = WritableUtils.readString(in);
  blockId = WritableUtils.readVLong(in);
  int length = WritableUtils.readVIntInRange(in, 0,
      AccessMode.class.getEnumConstants().length);
  for (int i = 0; i < length; i++) {
    modes.add(WritableUtils.readEnum(in, AccessMode.class));
  }
}
 

/**
 * Reads the values of each field.
 *
 * @param in  The input to read from.
 * @throws IOException When reading the input fails.
 */
@Override
public void readFields(DataInput in) throws IOException {
  Version version = Version.UNVERSIONED;
  // TableSplit was not versioned in the beginning.
  // In order to introduce it now, we make use of the fact
  // that tableName was written with Bytes.writeByteArray,
  // which encodes the array length as a vint which is >= 0.
  // Hence if the vint is >= 0 we have an old version and the vint
  // encodes the length of tableName.
  // If < 0 we just read the version and the next vint is the length.
  // @see Bytes#readByteArray(DataInput)
  int len = WritableUtils.readVInt(in);
  if (len < 0) {
    // what we just read was the version
    version = Version.fromCode(len);
    len = WritableUtils.readVInt(in);
  }
  byte[] tableNameBytes = new byte[len];
  in.readFully(tableNameBytes);
  tableName = TableName.valueOf(tableNameBytes);
  startRow = Bytes.readByteArray(in);
  endRow = Bytes.readByteArray(in);
  regionLocation = Bytes.toString(Bytes.readByteArray(in));
  if (version.atLeast(Version.INITIAL)) {
    scan = Bytes.toString(Bytes.readByteArray(in));
  }
  length = WritableUtils.readVLong(in);
  if (version.atLeast(Version.WITH_ENCODED_REGION_NAME)) {
    encodedRegionName = Bytes.toString(Bytes.readByteArray(in));
  }
}
 

@Override
public void readFields(DataInput input) throws IOException {
    super.readFields(input);
    divBy = WritableUtils.readVLong(input);
}
 

public void readFields(DataInput in) throws IOException {
  file = new Path(WritableUtils.readString(in));
  len = WritableUtils.readVLong(in);
}
 

public void readFields(DataInput in) throws IOException {
  segmentOffset = WritableUtils.readVLong(in);
  firstIndexId = WritableUtils.readVLong(in);
  firstIndexOffset = WritableUtils.readVLong(in);
  lastIndexOffset = WritableUtils.readVLong(in);
}
 

public void readFields(DataInput in) throws IOException {
  mapId = WritableUtils.readStringSafely(in, MAX_ID_LENGTH);
  compressedLength = WritableUtils.readVLong(in);
  uncompressedLength = WritableUtils.readVLong(in);
  forReduce = WritableUtils.readVInt(in);
}
 

@Override
public void fromBinary(DataInputStream in) throws IOException {
  value = WritableUtils.readVLong(in);
}
 

@Override
public void readFields(DataInput dataInput) throws IOException {
  rk = WritableUtils.readVLong(dataInput);
  next = WritableUtils.readVLong(dataInput);
}
 

/**
 * Reads a zero-compressed encoded long from a stream and return it.
 * @param in input stream
 * @throws java.io.IOException
 * @return deserialized long
 */
public static long readVLong(DataInput in) throws IOException {
  return WritableUtils.readVLong(in);
}