Java Code Examples for org.apache.hadoop.io.WritableUtils#readVLong()
The following examples show how to use
org.apache.hadoop.io.WritableUtils#readVLong() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TupleWritable.java From big-c with Apache License 2.0 | 6 votes |
/** * Reads a bitset from the stream that has been written with * {@link #writeBitSet(DataOutput, int, BitSet)}. */ private static final void readBitSet(DataInput stream, int nbits, BitSet bitSet) throws IOException { bitSet.clear(); long initialBits = WritableUtils.readVLong(stream); long last = 0L; while (0L != initialBits) { last = Long.lowestOneBit(initialBits); initialBits ^= last; bitSet.set(Long.numberOfTrailingZeros(last)); } for (int offset=Long.SIZE; offset < nbits; offset+=Byte.SIZE) { byte bits = stream.readByte(); while (0 != bits) { last = Long.lowestOneBit(bits); bits ^= last; bitSet.set(Long.numberOfTrailingZeros(last) + offset); } } }
Example 2
Source File: SequenceFile.java From gemfirexd-oss with Apache License 2.0 | 6 votes |
/** This constructor is there primarily to serve the sort routine that * generates a single output file with an associated index file */ public SegmentContainer(Path inName, Path indexIn) throws IOException { //get the segments from indexIn FSDataInputStream fsIndexIn = fs.open(indexIn); long end = fs.getFileStatus(indexIn).getLen(); while (fsIndexIn.getPos() < end) { long segmentOffset = WritableUtils.readVLong(fsIndexIn); long segmentLength = WritableUtils.readVLong(fsIndexIn); Path segmentName = inName; segments.add(new LinkedSegmentsDescriptor(segmentOffset, segmentLength, segmentName, this)); } fsIndexIn.close(); fs.delete(indexIn, true); numSegmentsContained = segments.size(); this.inName = inName; }
Example 3
Source File: SleepJob.java From big-c with Apache License 2.0 | 6 votes |
@Override public void readFields(DataInput in) throws IOException { id = WritableUtils.readVInt(in); sleepDuration = WritableUtils.readVLong(in); nMaps = WritableUtils.readVInt(in); nSpec = WritableUtils.readVInt(in); if (reduceDurations.length < nSpec) { reduceDurations = new long[nSpec]; } for (int i = 0; i < nSpec; ++i) { reduceDurations[i] = WritableUtils.readVLong(in); } final int nLoc = WritableUtils.readVInt(in); if (nLoc != locations.length) { locations = new String[nLoc]; } for (int i = 0; i < nLoc; ++i) { locations[i] = Text.readString(in); } }
Example 4
Source File: LoadSplit.java From big-c with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); id = WritableUtils.readVInt(in); maps = WritableUtils.readVInt(in); inputRecords = WritableUtils.readVLong(in); outputBytes = WritableUtils.readVLong(in); outputRecords = WritableUtils.readVLong(in); maxMemory = WritableUtils.readVLong(in); reduces = WritableUtils.readVInt(in); if (reduceBytes.length < reduces) { reduceBytes = new double[reduces]; reduceRecords = new double[reduces]; } for (int i = 0; i < reduces; ++i) { reduceBytes[i] = in.readDouble(); reduceRecords[i] = in.readDouble(); } nSpec = WritableUtils.readVInt(in); if (reduceOutputBytes.length < nSpec) { reduceOutputBytes = new long[nSpec]; reduceOutputRecords = new long[nSpec]; } for (int i = 0; i < nSpec; ++i) { reduceOutputBytes[i] = WritableUtils.readVLong(in); reduceOutputRecords[i] = WritableUtils.readVLong(in); } mapMetrics = new ResourceUsageMetrics(); mapMetrics.readFields(in); int numReduceMetrics = WritableUtils.readVInt(in); reduceMetrics = new ResourceUsageMetrics[numReduceMetrics]; for (int i = 0; i < numReduceMetrics; ++i) { reduceMetrics[i] = new ResourceUsageMetrics(); reduceMetrics[i].readFields(in); } }
Example 5
Source File: GridmixKey.java From big-c with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { rec_in = WritableUtils.readVLong(in); rec_out = WritableUtils.readVLong(in); bytes_out = WritableUtils.readVLong(in); sizeOfResourceUsageMetrics = WritableUtils.readVInt(in); if (sizeOfResourceUsageMetrics > 0) { metrics = new ResourceUsageMetrics(); metrics.readFields(in); } }
Example 6
Source File: PhoenixInputSplit.java From phoenix with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput input) throws IOException { regionLocation = WritableUtils.readString(input); regionSize = WritableUtils.readVLong(input); int count = WritableUtils.readVInt(input); scans = Lists.newArrayListWithExpectedSize(count); for (int i = 0; i < count; i++) { byte[] protoScanBytes = new byte[WritableUtils.readVInt(input)]; input.readFully(protoScanBytes); ClientProtos.Scan protoScan = ClientProtos.Scan.parseFrom(protoScanBytes); Scan scan = ProtobufUtil.toScan(protoScan); scans.add(scan); } init(); }
Example 7
Source File: GridmixKey.java From hadoop with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { rec_in = WritableUtils.readVLong(in); rec_out = WritableUtils.readVLong(in); bytes_out = WritableUtils.readVLong(in); sizeOfResourceUsageMetrics = WritableUtils.readVInt(in); if (sizeOfResourceUsageMetrics > 0) { metrics = new ResourceUsageMetrics(); metrics.readFields(in); } }
Example 8
Source File: GenericCounter.java From hadoop with Apache License 2.0 | 5 votes |
@Override public synchronized void readFields(DataInput in) throws IOException { name = StringInterner.weakIntern(Text.readString(in)); displayName = in.readBoolean() ? StringInterner.weakIntern(Text.readString(in)) : name; value = WritableUtils.readVLong(in); }
Example 9
Source File: LobFile.java From aliyun-maxcompute-data-collectors with Apache License 2.0 | 5 votes |
public void readFields(DataInput in) throws IOException { long recordTypeId = WritableUtils.readVLong(in); if (recordTypeId != INDEX_TABLE_ID) { // We expected to read an IndexTable. throw new IOException("Expected IndexTable; got record with typeId=" + recordTypeId); } int tableCount = WritableUtils.readVInt(in); tableEntries = new ArrayList<IndexTableEntry>(tableCount); for (int i = 0; i < tableCount; i++) { tableEntries.add(new IndexTableEntry(in)); } }
Example 10
Source File: BlockTokenSecretManager.java From hadoop with Apache License 2.0 | 5 votes |
/** * check if a token is expired. for unit test only. return true when token is * expired, false otherwise */ static boolean isTokenExpired(Token<BlockTokenIdentifier> token) throws IOException { ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier()); DataInputStream in = new DataInputStream(buf); long expiryDate = WritableUtils.readVLong(in); return isExpired(expiryDate); }
Example 11
Source File: LoadSplit.java From hadoop with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { super.readFields(in); id = WritableUtils.readVInt(in); maps = WritableUtils.readVInt(in); inputRecords = WritableUtils.readVLong(in); outputBytes = WritableUtils.readVLong(in); outputRecords = WritableUtils.readVLong(in); maxMemory = WritableUtils.readVLong(in); reduces = WritableUtils.readVInt(in); if (reduceBytes.length < reduces) { reduceBytes = new double[reduces]; reduceRecords = new double[reduces]; } for (int i = 0; i < reduces; ++i) { reduceBytes[i] = in.readDouble(); reduceRecords[i] = in.readDouble(); } nSpec = WritableUtils.readVInt(in); if (reduceOutputBytes.length < nSpec) { reduceOutputBytes = new long[nSpec]; reduceOutputRecords = new long[nSpec]; } for (int i = 0; i < nSpec; ++i) { reduceOutputBytes[i] = WritableUtils.readVLong(in); reduceOutputRecords[i] = WritableUtils.readVLong(in); } mapMetrics = new ResourceUsageMetrics(); mapMetrics.readFields(in); int numReduceMetrics = WritableUtils.readVInt(in); reduceMetrics = new ResourceUsageMetrics[numReduceMetrics]; for (int i = 0; i < numReduceMetrics; ++i) { reduceMetrics[i] = new ResourceUsageMetrics(); reduceMetrics[i].readFields(in); } }
Example 12
Source File: BlockTokenIdentifier.java From hadoop with Apache License 2.0 | 5 votes |
@Override public void readFields(DataInput in) throws IOException { this.cache = null; expiryDate = WritableUtils.readVLong(in); keyId = WritableUtils.readVInt(in); userId = WritableUtils.readString(in); blockPoolId = WritableUtils.readString(in); blockId = WritableUtils.readVLong(in); int length = WritableUtils.readVIntInRange(in, 0, AccessMode.class.getEnumConstants().length); for (int i = 0; i < length; i++) { modes.add(WritableUtils.readEnum(in, AccessMode.class)); } }
Example 13
Source File: TableSplit.java From hbase with Apache License 2.0 | 5 votes |
/** * Reads the values of each field. * * @param in The input to read from. * @throws IOException When reading the input fails. */ @Override public void readFields(DataInput in) throws IOException { Version version = Version.UNVERSIONED; // TableSplit was not versioned in the beginning. // In order to introduce it now, we make use of the fact // that tableName was written with Bytes.writeByteArray, // which encodes the array length as a vint which is >= 0. // Hence if the vint is >= 0 we have an old version and the vint // encodes the length of tableName. // If < 0 we just read the version and the next vint is the length. // @see Bytes#readByteArray(DataInput) int len = WritableUtils.readVInt(in); if (len < 0) { // what we just read was the version version = Version.fromCode(len); len = WritableUtils.readVInt(in); } byte[] tableNameBytes = new byte[len]; in.readFully(tableNameBytes); tableName = TableName.valueOf(tableNameBytes); startRow = Bytes.readByteArray(in); endRow = Bytes.readByteArray(in); regionLocation = Bytes.toString(Bytes.readByteArray(in)); if (version.atLeast(Version.INITIAL)) { scan = Bytes.toString(Bytes.readByteArray(in)); } length = WritableUtils.readVLong(in); if (version.atLeast(Version.WITH_ENCODED_REGION_NAME)) { encodedRegionName = Bytes.toString(Bytes.readByteArray(in)); } }
Example 14
Source File: RoundDateExpression.java From phoenix with BSD 3-Clause "New" or "Revised" License | 4 votes |
@Override public void readFields(DataInput input) throws IOException { super.readFields(input); divBy = WritableUtils.readVLong(input); }
Example 15
Source File: GenericMRLoadGenerator.java From hadoop with Apache License 2.0 | 4 votes |
public void readFields(DataInput in) throws IOException { file = new Path(WritableUtils.readString(in)); len = WritableUtils.readVLong(in); }
Example 16
Source File: LobFile.java From aliyun-maxcompute-data-collectors with Apache License 2.0 | 4 votes |
public void readFields(DataInput in) throws IOException { segmentOffset = WritableUtils.readVLong(in); firstIndexId = WritableUtils.readVLong(in); firstIndexOffset = WritableUtils.readVLong(in); lastIndexOffset = WritableUtils.readVLong(in); }
Example 17
Source File: ShuffleHeader.java From hadoop with Apache License 2.0 | 4 votes |
public void readFields(DataInput in) throws IOException { mapId = WritableUtils.readStringSafely(in, MAX_ID_LENGTH); compressedLength = WritableUtils.readVLong(in); uncompressedLength = WritableUtils.readVLong(in); forReduce = WritableUtils.readVInt(in); }
Example 18
Source File: Tokenizer.java From RDFS with Apache License 2.0 | 4 votes |
@Override public void fromBinary(DataInputStream in) throws IOException { value = WritableUtils.readVLong(in); }
Example 19
Source File: IntegrationTestBulkLoad.java From hbase with Apache License 2.0 | 4 votes |
@Override public void readFields(DataInput dataInput) throws IOException { rk = WritableUtils.readVLong(dataInput); next = WritableUtils.readVLong(dataInput); }
Example 20
Source File: Utils.java From RDFS with Apache License 2.0 | 2 votes |
/** * Reads a zero-compressed encoded long from a stream and return it. * @param in input stream * @throws java.io.IOException * @return deserialized long */ public static long readVLong(DataInput in) throws IOException { return WritableUtils.readVLong(in); }