Java Code Examples for org.apache.hadoop.io.WritableUtils#writeVLong()

The following examples show how to use org.apache.hadoop.io.WritableUtils#writeVLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LoadSplit.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void write(DataOutput out) throws IOException {
  super.write(out);
  WritableUtils.writeVInt(out, id);
  WritableUtils.writeVInt(out, maps);
  WritableUtils.writeVLong(out, inputRecords);
  WritableUtils.writeVLong(out, outputBytes);
  WritableUtils.writeVLong(out, outputRecords);
  WritableUtils.writeVLong(out, maxMemory);
  WritableUtils.writeVInt(out, reduces);
  for (int i = 0; i < reduces; ++i) {
    out.writeDouble(reduceBytes[i]);
    out.writeDouble(reduceRecords[i]);
  }
  WritableUtils.writeVInt(out, nSpec);
  for (int i = 0; i < nSpec; ++i) {
    WritableUtils.writeVLong(out, reduceOutputBytes[i]);
    WritableUtils.writeVLong(out, reduceOutputRecords[i]);
  }
  mapMetrics.write(out);
  int numReduceMetrics = (reduceMetrics == null) ? 0 : reduceMetrics.length;
  WritableUtils.writeVInt(out, numReduceMetrics);
  for (int i = 0; i < numReduceMetrics; ++i) {
    reduceMetrics[i].write(out);
  }
}
 
Example 2
Source File: FSImageSerialization.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Write an array of blocks as compactly as possible. This uses
 * delta-encoding for the generation stamp and size, following
 * the principle that genstamp increases relatively slowly,
 * and size is equal for all but the last block of a file.
 */
public static void writeCompactBlockArray(
    Block[] blocks, DataOutputStream out) throws IOException {
  WritableUtils.writeVInt(out, blocks.length);
  Block prev = null;
  for (Block b : blocks) {
    long szDelta = b.getNumBytes() -
        (prev != null ? prev.getNumBytes() : 0);
    long gsDelta = b.getGenerationStamp() -
        (prev != null ? prev.getGenerationStamp() : 0);
    out.writeLong(b.getBlockId()); // blockid is random
    WritableUtils.writeVLong(out, szDelta);
    WritableUtils.writeVLong(out, gsDelta);
    prev = b;
  }
}
 
Example 3
Source File: FrameworkCounterGroup.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * FrameworkGroup ::= #counter (key value)*
 */
@Override
@SuppressWarnings("unchecked")
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, size());
  for (int i = 0; i < counters.length; ++i) {
    TezCounter counter = (C) counters[i];
    if (counter != null) {
      WritableUtils.writeVInt(out, i);
      WritableUtils.writeVLong(out, counter.getValue());
    }
  }
}
 
Example 4
Source File: TupleWritable.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
/** Writes each Writable to <code>out</code>.
 * TupleWritable format:
 * {@code
 *  <count><type1><type2>...<typen><obj1><obj2>...<objn>
 * }
 */
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, values.length);
  WritableUtils.writeVLong(out, written);
  for (int i = 0; i < values.length; ++i) {
    Text.writeString(out, values[i].getClass().getName());
  }
  for (int i = 0; i < values.length; ++i) {
    if (has(i)) {
      values[i].write(out);
    }
  }
}
 
Example 5
Source File: SequenceFile.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
private void flush(int count, int bytesProcessed, 
                   CompressionType compressionType, 
                   CompressionCodec codec, 
                   boolean done) throws IOException {
  if (out == null) {
    outName = done ? outFile : outFile.suffix(".0");
    out = fs.create(outName);
    if (!done) {
      indexOut = fs.create(outName.suffix(".index"));
    }
  }

  long segmentStart = out.getPos();
  Writer writer = createWriter(conf, Writer.stream(out), 
      Writer.keyClass(keyClass), Writer.valueClass(valClass),
      Writer.compression(compressionType, codec),
      Writer.metadata(done ? metadata : new Metadata()));
  
  if (!done) {
    writer.sync = null;                     // disable sync on temp files
  }

  for (int i = 0; i < count; i++) {         // write in sorted order
    int p = pointers[i];
    writer.appendRaw(rawBuffer, keyOffsets[p], keyLengths[p], rawValues[p]);
  }
  writer.close();
  
  if (!done) {
    // Save the segment length
    WritableUtils.writeVLong(indexOut, segmentStart);
    WritableUtils.writeVLong(indexOut, (out.getPos()-segmentStart));
    indexOut.flush();
  }
}
 
Example 6
Source File: DelegationKey.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 */
@Override
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, keyId);
  WritableUtils.writeVLong(out, expiryDate);
  if (keyBytes == null) {
    WritableUtils.writeVInt(out, -1);
  } else {
    WritableUtils.writeVInt(out, keyBytes.length);
    out.write(keyBytes);
  }
}
 
Example 7
Source File: TupleWritable.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Writes the bit set to the stream. The first 64 bit-positions of the bit
 * set are written as a VLong for backwards-compatibility with older 
 * versions of TupleWritable. All bit-positions >= 64 are encoded as a byte
 * for every 8 bit-positions.
 */
private static final void writeBitSet(DataOutput stream, int nbits,
    BitSet bitSet) throws IOException {
  long bits = 0L;
      
  int bitSetIndex = bitSet.nextSetBit(0);
  for (;bitSetIndex >= 0 && bitSetIndex < Long.SIZE;
          bitSetIndex=bitSet.nextSetBit(bitSetIndex+1)) {
    bits |= 1L << bitSetIndex;
  }
  WritableUtils.writeVLong(stream,bits);
  
  if (nbits > Long.SIZE) {
    bits = 0L;
    for (int lastWordWritten = 0; bitSetIndex >= 0 && bitSetIndex < nbits; 
            bitSetIndex = bitSet.nextSetBit(bitSetIndex+1)) {
      int bitsIndex = bitSetIndex % Byte.SIZE;
      int word = (bitSetIndex-Long.SIZE) / Byte.SIZE;
      if (word > lastWordWritten) {
        stream.writeByte((byte)bits);
        bits = 0L;
        for (lastWordWritten++;lastWordWritten<word;lastWordWritten++) {
          stream.writeByte((byte)bits);
        }
      }
      bits |= 1L << bitsIndex;
    }
    stream.writeByte((byte)bits);
  }
}
 
Example 8
Source File: EncodedQualifiersColumnProjectionFilter.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void write(DataOutput output) throws IOException {
    WritableUtils.writeCompressedByteArray(output, this.emptyCFName);
    long[] longArrayOfBitSet = trackedColumns.toLongArray();
    WritableUtils.writeVInt(output, longArrayOfBitSet.length);
    for (Long l : longArrayOfBitSet) {
        WritableUtils.writeVLong(output, l);
    }
    WritableUtils.writeVInt(output, encodingScheme.ordinal());
    WritableUtils.writeVInt(output, this.conditionOnlyCfs.size());
    for (byte[] f : this.conditionOnlyCfs) {
        WritableUtils.writeCompressedByteArray(output, f);
    }
}
 
Example 9
Source File: TeraGen.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVLong(out, firstRow);
  WritableUtils.writeVLong(out, rowCount);
}
 
Example 10
Source File: Tokenizer.java    From RDFS with Apache License 2.0 4 votes vote down vote up
@Override
public void toBinary(DataOutputStream out) throws IOException {
  WritableUtils.writeVLong(out, value);
}
 
Example 11
Source File: SortValidator.java    From RDFS with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVLong(out, bytes);
  WritableUtils.writeVLong(out, records);
  WritableUtils.writeVInt(out, checksum);
}
 
Example 12
Source File: ShuffleHeader.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  Text.writeString(out, mapId);
  WritableUtils.writeVLong(out, compressedLength);
  WritableUtils.writeVLong(out, uncompressedLength);
  WritableUtils.writeVInt(out, forReduce);
}
 
Example 13
Source File: IntegrationTestBulkLoad.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void write(DataOutput dataOutput) throws IOException {
  WritableUtils.writeVLong(dataOutput, chainId);
  WritableUtils.writeVLong(dataOutput, order);
}
 
Example 14
Source File: LobFile.java    From aliyun-maxcompute-data-collectors with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVLong(out, segmentOffset);
  WritableUtils.writeVLong(out, firstIndexId);
  WritableUtils.writeVLong(out, firstIndexOffset);
  WritableUtils.writeVLong(out, lastIndexOffset);
}
 
Example 15
Source File: TextLongWritable.java    From 163-bigdate-note with GNU General Public License v3.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
    this.text.write(out);
    WritableUtils.writeVLong(out, this.compareValue.get());
}
 
Example 16
Source File: RoundDateExpression.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
public void write(DataOutput output) throws IOException {
    super.write(output);
    WritableUtils.writeVLong(output, divBy);
}
 
Example 17
Source File: GenericMRLoadGenerator.java    From hadoop-gpu with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  WritableUtils.writeString(out, file.toString());
  WritableUtils.writeVLong(out, len);
}
 
Example 18
Source File: TeraGen.java    From hadoop-book with Apache License 2.0 4 votes vote down vote up
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVLong(out, firstRow);
  WritableUtils.writeVLong(out, rowCount);
}
 
Example 19
Source File: Utils.java    From hadoop-gpu with Apache License 2.0 2 votes vote down vote up
/**
 * Serializes a long to a binary stream with zero-compressed encoding.
 * For -112 <= i <= 127, only one byte is used with the actual value.
 * For other values of i, the first byte value indicates whether the
 * long is positive or negative, and the number of bytes that follow.
 * If the first byte value v is between -113 and -120, the following long
 * is positive, with number of bytes that follow are -(v+112).
 * If the first byte value v is between -121 and -128, the following long
 * is negative, with number of bytes that follow are -(v+120). Bytes are
 * stored in the high-non-zero-byte-first order.
 *
 * @param stream Binary output stream
 * @param i Long to be serialized
 * @throws java.io.IOException
 */
public static void writeVLong(DataOutput stream, long i) throws IOException {
  WritableUtils.writeVLong(stream, i);
}
 
Example 20
Source File: Utils.java    From hadoop with Apache License 2.0 2 votes vote down vote up
/**
 * Serializes a long to a binary stream with zero-compressed encoding.
 * For {@literal -112 <= i <= 127}, only one byte is used with the actual
 * value. For other values of i, the first byte value indicates whether the
 * long is positive or negative, and the number of bytes that follow.
 * If the first byte value v is between -113 and -120, the following long
 * is positive, with number of bytes that follow are -(v+112).
 * If the first byte value v is between -121 and -128, the following long
 * is negative, with number of bytes that follow are -(v+120). Bytes are
 * stored in the high-non-zero-byte-first order.
 *
 * @param stream Binary output stream
 * @param i Long to be serialized
 * @throws java.io.IOException
 */
public static void writeVLong(DataOutput stream, long i) throws IOException {
  WritableUtils.writeVLong(stream, i);
}