Java Code Examples for org.apache.hadoop.io.WritableUtils#readVLong()

The following examples show how to use org.apache.hadoop.io.WritableUtils#readVLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TupleWritable.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Reads a bitset from the stream that has been written with
 * {@link #writeBitSet(DataOutput, int, BitSet)}.
 */
private static final void readBitSet(DataInput stream, int nbits, 
    BitSet bitSet) throws IOException {
  bitSet.clear();
  long initialBits = WritableUtils.readVLong(stream);
  long last = 0L;
  while (0L != initialBits) {
    last = Long.lowestOneBit(initialBits);
    initialBits ^= last;
    bitSet.set(Long.numberOfTrailingZeros(last));
  }
  
  for (int offset=Long.SIZE; offset < nbits; offset+=Byte.SIZE) {
    byte bits = stream.readByte();
    while (0 != bits) {
      last = Long.lowestOneBit(bits);
      bits ^= last;
      bitSet.set(Long.numberOfTrailingZeros(last) + offset);
    }
  }
}
 
Example 2
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 6 votes vote down vote up
/** This constructor is there primarily to serve the sort routine that 
 * generates a single output file with an associated index file */
public SegmentContainer(Path inName, Path indexIn) throws IOException {
  //get the segments from indexIn
  FSDataInputStream fsIndexIn = fs.open(indexIn);
  long end = fs.getFileStatus(indexIn).getLen();
  while (fsIndexIn.getPos() < end) {
    long segmentOffset = WritableUtils.readVLong(fsIndexIn);
    long segmentLength = WritableUtils.readVLong(fsIndexIn);
    Path segmentName = inName;
    segments.add(new LinkedSegmentsDescriptor(segmentOffset, 
                                              segmentLength, segmentName, this));
  }
  fsIndexIn.close();
  fs.delete(indexIn, true);
  numSegmentsContained = segments.size();
  this.inName = inName;
}
 
Example 3
Source File: SleepJob.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  id = WritableUtils.readVInt(in);
  sleepDuration = WritableUtils.readVLong(in);
  nMaps = WritableUtils.readVInt(in);
  nSpec = WritableUtils.readVInt(in);
  if (reduceDurations.length < nSpec) {
    reduceDurations = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceDurations[i] = WritableUtils.readVLong(in);
  }
  final int nLoc = WritableUtils.readVInt(in);
  if (nLoc != locations.length) {
    locations = new String[nLoc];
  }
  for (int i = 0; i < nLoc; ++i) {
    locations[i] = Text.readString(in);
  }
}
 
Example 4
Source File: LoadSplit.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  super.readFields(in);
  id = WritableUtils.readVInt(in);
  maps = WritableUtils.readVInt(in);
  inputRecords = WritableUtils.readVLong(in);
  outputBytes = WritableUtils.readVLong(in);
  outputRecords = WritableUtils.readVLong(in);
  maxMemory = WritableUtils.readVLong(in);
  reduces = WritableUtils.readVInt(in);
  if (reduceBytes.length < reduces) {
    reduceBytes = new double[reduces];
    reduceRecords = new double[reduces];
  }
  for (int i = 0; i < reduces; ++i) {
    reduceBytes[i] = in.readDouble();
    reduceRecords[i] = in.readDouble();
  }
  nSpec = WritableUtils.readVInt(in);
  if (reduceOutputBytes.length < nSpec) {
    reduceOutputBytes = new long[nSpec];
    reduceOutputRecords = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceOutputBytes[i] = WritableUtils.readVLong(in);
    reduceOutputRecords[i] = WritableUtils.readVLong(in);
  }
  mapMetrics = new ResourceUsageMetrics();
  mapMetrics.readFields(in);
  int numReduceMetrics = WritableUtils.readVInt(in);
  reduceMetrics = new ResourceUsageMetrics[numReduceMetrics];
  for (int i = 0; i < numReduceMetrics; ++i) {
    reduceMetrics[i] = new ResourceUsageMetrics();
    reduceMetrics[i].readFields(in);
  }
}
 
Example 5
Source File: GridmixKey.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  rec_in = WritableUtils.readVLong(in);
  rec_out = WritableUtils.readVLong(in);
  bytes_out = WritableUtils.readVLong(in);
  sizeOfResourceUsageMetrics =  WritableUtils.readVInt(in);
  if (sizeOfResourceUsageMetrics > 0) {
    metrics = new ResourceUsageMetrics();
    metrics.readFields(in);
  }
}
 
Example 6
Source File: PhoenixInputSplit.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput input) throws IOException {
    regionLocation = WritableUtils.readString(input);
    regionSize = WritableUtils.readVLong(input);
    int count = WritableUtils.readVInt(input);
    scans = Lists.newArrayListWithExpectedSize(count);
    for (int i = 0; i < count; i++) {
        byte[] protoScanBytes = new byte[WritableUtils.readVInt(input)];
        input.readFully(protoScanBytes);
        ClientProtos.Scan protoScan = ClientProtos.Scan.parseFrom(protoScanBytes);
        Scan scan = ProtobufUtil.toScan(protoScan);
        scans.add(scan);
    }
    init();
}
 
Example 7
Source File: GridmixKey.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  rec_in = WritableUtils.readVLong(in);
  rec_out = WritableUtils.readVLong(in);
  bytes_out = WritableUtils.readVLong(in);
  sizeOfResourceUsageMetrics =  WritableUtils.readVInt(in);
  if (sizeOfResourceUsageMetrics > 0) {
    metrics = new ResourceUsageMetrics();
    metrics.readFields(in);
  }
}
 
Example 8
Source File: GenericCounter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void readFields(DataInput in) throws IOException {
  name = StringInterner.weakIntern(Text.readString(in));
  displayName = in.readBoolean() ? 
      StringInterner.weakIntern(Text.readString(in)) : name;
  value = WritableUtils.readVLong(in);
}
 
Example 9
Source File: LobFile.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  long recordTypeId = WritableUtils.readVLong(in);
  if (recordTypeId != INDEX_TABLE_ID) {
    // We expected to read an IndexTable.
    throw new IOException("Expected IndexTable; got record with typeId="
        + recordTypeId);
  }

  int tableCount = WritableUtils.readVInt(in);

  tableEntries = new ArrayList<IndexTableEntry>(tableCount);
  for (int i = 0; i < tableCount; i++) {
    tableEntries.add(new IndexTableEntry(in));
  }
}
 
Example 10
Source File: BlockTokenSecretManager.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * check if a token is expired. for unit test only. return true when token is
 * expired, false otherwise
 */
static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
    throws IOException {
  ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
  DataInputStream in = new DataInputStream(buf);
  long expiryDate = WritableUtils.readVLong(in);
  return isExpired(expiryDate);
}
 
Example 11
Source File: LoadSplit.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  super.readFields(in);
  id = WritableUtils.readVInt(in);
  maps = WritableUtils.readVInt(in);
  inputRecords = WritableUtils.readVLong(in);
  outputBytes = WritableUtils.readVLong(in);
  outputRecords = WritableUtils.readVLong(in);
  maxMemory = WritableUtils.readVLong(in);
  reduces = WritableUtils.readVInt(in);
  if (reduceBytes.length < reduces) {
    reduceBytes = new double[reduces];
    reduceRecords = new double[reduces];
  }
  for (int i = 0; i < reduces; ++i) {
    reduceBytes[i] = in.readDouble();
    reduceRecords[i] = in.readDouble();
  }
  nSpec = WritableUtils.readVInt(in);
  if (reduceOutputBytes.length < nSpec) {
    reduceOutputBytes = new long[nSpec];
    reduceOutputRecords = new long[nSpec];
  }
  for (int i = 0; i < nSpec; ++i) {
    reduceOutputBytes[i] = WritableUtils.readVLong(in);
    reduceOutputRecords[i] = WritableUtils.readVLong(in);
  }
  mapMetrics = new ResourceUsageMetrics();
  mapMetrics.readFields(in);
  int numReduceMetrics = WritableUtils.readVInt(in);
  reduceMetrics = new ResourceUsageMetrics[numReduceMetrics];
  for (int i = 0; i < numReduceMetrics; ++i) {
    reduceMetrics[i] = new ResourceUsageMetrics();
    reduceMetrics[i].readFields(in);
  }
}
 
Example 12
Source File: BlockTokenIdentifier.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  this.cache = null;
  expiryDate = WritableUtils.readVLong(in);
  keyId = WritableUtils.readVInt(in);
  userId = WritableUtils.readString(in);
  blockPoolId = WritableUtils.readString(in);
  blockId = WritableUtils.readVLong(in);
  int length = WritableUtils.readVIntInRange(in, 0,
      AccessMode.class.getEnumConstants().length);
  for (int i = 0; i < length; i++) {
    modes.add(WritableUtils.readEnum(in, AccessMode.class));
  }
}
 
Example 13
Source File: TableSplit.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Reads the values of each field.
 *
 * @param in  The input to read from.
 * @throws IOException When reading the input fails.
 */
@Override
public void readFields(DataInput in) throws IOException {
  Version version = Version.UNVERSIONED;
  // TableSplit was not versioned in the beginning.
  // In order to introduce it now, we make use of the fact
  // that tableName was written with Bytes.writeByteArray,
  // which encodes the array length as a vint which is >= 0.
  // Hence if the vint is >= 0 we have an old version and the vint
  // encodes the length of tableName.
  // If < 0 we just read the version and the next vint is the length.
  // @see Bytes#readByteArray(DataInput)
  int len = WritableUtils.readVInt(in);
  if (len < 0) {
    // what we just read was the version
    version = Version.fromCode(len);
    len = WritableUtils.readVInt(in);
  }
  byte[] tableNameBytes = new byte[len];
  in.readFully(tableNameBytes);
  tableName = TableName.valueOf(tableNameBytes);
  startRow = Bytes.readByteArray(in);
  endRow = Bytes.readByteArray(in);
  regionLocation = Bytes.toString(Bytes.readByteArray(in));
  if (version.atLeast(Version.INITIAL)) {
    scan = Bytes.toString(Bytes.readByteArray(in));
  }
  length = WritableUtils.readVLong(in);
  if (version.atLeast(Version.WITH_ENCODED_REGION_NAME)) {
    encodedRegionName = Bytes.toString(Bytes.readByteArray(in));
  }
}
 
Example 14
Source File: RoundDateExpression.java    From phoenix with BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
@Override
public void readFields(DataInput input) throws IOException {
    super.readFields(input);
    divBy = WritableUtils.readVLong(input);
}
 
Example 15
Source File: GenericMRLoadGenerator.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  file = new Path(WritableUtils.readString(in));
  len = WritableUtils.readVLong(in);
}
 
Example 16
Source File: LobFile.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 4 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  segmentOffset = WritableUtils.readVLong(in);
  firstIndexId = WritableUtils.readVLong(in);
  firstIndexOffset = WritableUtils.readVLong(in);
  lastIndexOffset = WritableUtils.readVLong(in);
}
 
Example 17
Source File: ShuffleHeader.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  mapId = WritableUtils.readStringSafely(in, MAX_ID_LENGTH);
  compressedLength = WritableUtils.readVLong(in);
  uncompressedLength = WritableUtils.readVLong(in);
  forReduce = WritableUtils.readVInt(in);
}
 
Example 18
Source File: Tokenizer.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public void fromBinary(DataInputStream in) throws IOException {
  value = WritableUtils.readVLong(in);
}
 
Example 19
Source File: IntegrationTestBulkLoad.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void readFields(DataInput dataInput) throws IOException {
  rk = WritableUtils.readVLong(dataInput);
  next = WritableUtils.readVLong(dataInput);
}
 
Example 20
Source File: Utils.java    From RDFS with Apache License 2.0 2 votes vote down vote up
/**
 * Reads a zero-compressed encoded long from a stream and return it.
 * @param in input stream
 * @throws java.io.IOException
 * @return deserialized long
 */
public static long readVLong(DataInput in) throws IOException {
  return WritableUtils.readVLong(in);
}