org.apache.hadoop.io.WritableUtils Java Examples

The following examples show how to use org.apache.hadoop.io.WritableUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: GridmixRecord.java    From RDFS with Apache License 2.0 6 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  size = WritableUtils.readVInt(in);
  int payload = size - WritableUtils.getVIntSize(size);
  if (payload > Long.SIZE / Byte.SIZE) {
    seed = in.readLong();
    payload -= Long.SIZE / Byte.SIZE;
  } else {
    Arrays.fill(literal, (byte)0);
    in.readFully(literal, 0, payload);
    dib.reset(literal, 0, literal.length);
    seed = dib.readLong();
    payload = 0;
  }
  final int vBytes = in.skipBytes(payload);
  if (vBytes != payload) {
    throw new EOFException("Expected " + payload + ", read " + vBytes);
  }
}
 
Example #2
Source File: DatanodeInfo.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
public void readFields(DataInput in) throws IOException {
  super.readFields(in);

  //TODO: move it to DatanodeID once DatanodeID is not stored in FSImage
  this.ipcPort = in.readShort() & 0x0000ffff;

  this.capacity = in.readLong();
  this.dfsUsed = in.readLong();
  this.remaining = in.readLong();
  this.lastUpdate = in.readLong();
  this.xceiverCount = in.readInt();
  this.location = Text.readString(in);
  this.hostName = Text.readString(in);
  setAdminState(WritableUtils.readEnum(in, AdminStates.class));
}
 
Example #3
Source File: ColumnExpression.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public void readFields(DataInput input) throws IOException {
    // read/write type ordinal, maxLength presence, scale presence and isNullable bit together to save space
    int typeAndFlag = WritableUtils.readVInt(input);
    isNullable = (typeAndFlag & 0x01) != 0;
    if ((typeAndFlag & 0x02) != 0) {
        scale = WritableUtils.readVInt(input);
    }
    if ((typeAndFlag & 0x04) != 0) {
        maxLength = WritableUtils.readVInt(input);
    }
    type = PDataType.values()[typeAndFlag >>> 3];
    if (type.isFixedWidth() && type.getByteSize() == null) {
        byteSize = WritableUtils.readVInt(input);
    }
    columnModifier = ColumnModifier.fromSystemValue(WritableUtils.readVInt(input));
}
 
Example #4
Source File: TestTFileStreams.java    From attic-apex-malhar with Apache License 2.0 6 votes vote down vote up
private long writeRecords(int count, boolean knownKeyLength,
    boolean knownValueLength, boolean close) throws IOException {
  long rawDataSize = 0;
  for (int nx = 0; nx < count; nx++) {
    String key = TestDTFileByteArrays.composeSortedKey("key", nx);
    DataOutputStream outKey =
        writer.prepareAppendKey(knownKeyLength ? key.length() : -1);
    outKey.write(key.getBytes());
    outKey.close();
    String value = "value" + nx;
    DataOutputStream outValue =
        writer.prepareAppendValue(knownValueLength ? value.length() : -1);
    outValue.write(value.getBytes());
    outValue.close();
    rawDataSize +=
        WritableUtils.getVIntSize(key.getBytes().length)
            + key.getBytes().length
            + WritableUtils.getVIntSize(value.getBytes().length)
            + value.getBytes().length;
  }
  if (close) {
    closeOutput();
  }
  return rawDataSize;
}
 
Example #5
Source File: QueueInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void write(DataOutput out) throws IOException {
  Text.writeString(out, queueName);
  WritableUtils.writeEnum(out, queueState);
  
  if(schedulingInfo!= null) {
    Text.writeString(out, schedulingInfo);
  }else {
    Text.writeString(out, "N/A");
  }
  out.writeInt(stats.length);
  for (JobStatus stat : stats) {
    stat.write(out);
  }
  out.writeInt(children.size());
  for(QueueInfo childQueueInfo : children) {
    childQueueInfo.write(out);
  }
}
 
Example #6
Source File: ReducePartition.java    From RDFS with Apache License 2.0 6 votes vote down vote up
public boolean next(DataInputBuffer key, DataInputBuffer value)
    throws IOException {
  MemoryBlockIndex memBlkIdx = keyValueIterator.next();
  if (memBlkIdx != null) {
    int pos = memBlkIdx.getIndex();
    MemoryBlock memBlk = memBlkIdx.getMemoryBlock();
    int offset = memBlk.offsets[pos];
    int keyLen = memBlk.keyLenArray[pos];
    int valLen = memBlk.valueLenArray[pos];
    dataOutputBuffer.reset();
    dataOutputBuffer.writeInt(keyLen);
    dataOutputBuffer.write(kvbuffer, offset, keyLen);
    dataOutputBuffer.writeInt(valLen);
    dataOutputBuffer.write(kvbuffer, offset + keyLen, valLen);
    key.reset(dataOutputBuffer.getData(), 0, keyLen
        + WritableUtils.INT_LENGTH_BYTES);
    value.reset(dataOutputBuffer.getData(), keyLen
        + WritableUtils.INT_LENGTH_BYTES, valLen
        + WritableUtils.INT_LENGTH_BYTES);
    return true;
  }
  return false;
}
 
Example #7
Source File: ByteUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 
Example #8
Source File: FileSystemCounterGroup.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * FileSystemGroup ::= #scheme (scheme #counter (key value)*)*
 */
@Override
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, map.size()); // #scheme
  for (Map.Entry<String, Object[]> entry : map.entrySet()) {
    WritableUtils.writeString(out, entry.getKey()); // scheme
    // #counter for the above scheme
    WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
    for (Object counter : entry.getValue()) {
      if (counter == null) continue;
      @SuppressWarnings("unchecked")
      FSCounter c = (FSCounter) ((Counter)counter).getUnderlyingCounter();
      WritableUtils.writeVInt(out, c.key.ordinal());  // key
      WritableUtils.writeVLong(out, c.getValue());    // value
    }
  }
}
 
Example #9
Source File: Task.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public void write(DataOutput out) throws IOException {
  Text.writeString(out, jobFile);
  taskId.write(out);
  out.writeInt(partition);
  out.writeInt(numSlotsRequired);
  taskStatus.write(out);
  skipRanges.write(out);
  out.writeBoolean(skipping);
  out.writeBoolean(jobCleanup);
  if (jobCleanup) {
    WritableUtils.writeEnum(out, jobRunStateForCleanup);
  }
  out.writeBoolean(jobSetup);
  out.writeBoolean(writeSkipRecs);
  out.writeBoolean(taskCleanup);
  Text.writeString(out, user);
  out.writeInt(encryptedSpillKey.length);
  extraData.write(out);
  out.write(encryptedSpillKey);
}
 
Example #10
Source File: Row.java    From emodb with Apache License 2.0 6 votes vote down vote up
/**
 * Sets this instance's content from the input.
 */
@Override
public void readFields(DataInput in)
        throws IOException {
    // Read the length as a variable int
    int length = WritableUtils.readVInt(in);
    // If necessary increase the buffer capacity
    if (length > _text.capacity()) {
        _text = ByteBuffer.allocate(length);
    }
    // For efficiency read directly into the buffer's array
    in.readFully(_text.array(), 0, length);
    // Since we bypassed putting into the buffer set the position and limit directly
    _text.position(0);
    _text.limit(length);
    // Set the map to null since the contents may have changed.
    _map = null;
}
 
Example #11
Source File: TaskReport.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void write(DataOutput out) throws IOException {
  taskid.write(out);
  out.writeFloat(progress);
  Text.writeString(out, state);
  out.writeLong(startTime);
  out.writeLong(finishTime);
  out.writeBoolean(runOnGPU);
  WritableUtils.writeStringArray(out, diagnostics);
  counters.write(out);
  WritableUtils.writeEnum(out, currentStatus);
  if (currentStatus == TIPStatus.RUNNING) {
    WritableUtils.writeVInt(out, runningAttempts.size());
    TaskAttemptID t[] = new TaskAttemptID[0];
    t = runningAttempts.toArray(t);
    for (int i = 0; i < t.length; i++) {
      t[i].write(out);
    }
  } else if (currentStatus == TIPStatus.COMPLETE) {
    successfulAttempt.write(out);
  }
}
 
Example #12
Source File: IFile.java    From big-c with Apache License 2.0 6 votes vote down vote up
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
 
Example #13
Source File: IFile.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void append(DataInputBuffer key, DataInputBuffer value)
throws IOException {
  int keyLength = key.getLength() - key.getPosition();
  if (keyLength < 0) {
    throw new IOException("Negative key-length not allowed: " + keyLength + 
                          " for " + key);
  }
  
  int valueLength = value.getLength() - value.getPosition();
  if (valueLength < 0) {
    throw new IOException("Negative value-length not allowed: " + 
                          valueLength + " for " + value);
  }

  WritableUtils.writeVInt(out, keyLength);
  WritableUtils.writeVInt(out, valueLength);
  out.write(key.getData(), key.getPosition(), keyLength); 
  out.write(value.getData(), value.getPosition(), valueLength); 

  // Update bytes written
  decompressedBytesWritten += keyLength + valueLength + 
                  WritableUtils.getVIntSize(keyLength) + 
                  WritableUtils.getVIntSize(valueLength);
  ++numRecordsWritten;
}
 
Example #14
Source File: DatanodeInfo.java    From RDFS with Apache License 2.0 6 votes vote down vote up
/** {@inheritDoc} */
public void write(DataOutput out) throws IOException {
  super.write(out);

  //TODO: move it to DatanodeID once DatanodeID is not stored in FSImage
  out.writeShort(ipcPort);

  out.writeLong(capacity);
  out.writeLong(dfsUsed);
  out.writeLong(remaining);
  out.writeLong(lastUpdate);
  out.writeInt(xceiverCount);
  Text.writeString(out, location);
  Text.writeString(out, hostName == null? "": hostName);
  WritableUtils.writeEnum(out, getAdminState());
}
 
Example #15
Source File: FileSystemCounterGroup.java    From tez with Apache License 2.0 6 votes vote down vote up
/**
 * FileSystemGroup ::= #scheme (scheme #counter (key value)*)*
 */
@Override
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, map.size()); // #scheme
  for (Map.Entry<String, Object[]> entry : map.entrySet()) {
    WritableUtils.writeString(out, entry.getKey()); // scheme
    // #counter for the above scheme
    WritableUtils.writeVInt(out, numSetCounters(entry.getValue()));
    for (Object counter : entry.getValue()) {
      if (counter == null) continue;
      FSCounter c = (FSCounter) ((TezCounter)counter).getUnderlyingCounter();
      WritableUtils.writeVInt(out, c.key.ordinal());  // key
      WritableUtils.writeVLong(out, c.getValue());    // value
    }
  }
}
 
Example #16
Source File: LinkDumper.java    From anthelion with Apache License 2.0 6 votes vote down vote up
/**
 * Aggregate all LinkNode objects for a given url.
 */
public void reduce(Text key, Iterator<LinkNode> values,
  OutputCollector<Text, LinkNodes> output, Reporter reporter)
  throws IOException {

  List<LinkNode> nodeList = new ArrayList<LinkNode>();
  int numNodes = 0;

  while (values.hasNext()) {
    LinkNode cur = values.next();
    if (numNodes < maxInlinks) {
      nodeList.add((LinkNode)WritableUtils.clone(cur, conf));
      numNodes++;
    }
    else {
      break;
    }
  }

  LinkNode[] linkNodesAr = nodeList.toArray(new LinkNode[nodeList.size()]);
  LinkNodes linkNodes = new LinkNodes(linkNodesAr);
  output.collect(key, linkNodes);
}
 
Example #17
Source File: CompositeInputSplit.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 * @throws IOException If the child InputSplit cannot be read, typically
 *                     for faliing access checks.
 */
@SuppressWarnings("unchecked")  // Generic array assignment
public void readFields(DataInput in) throws IOException {
  int card = WritableUtils.readVInt(in);
  if (splits == null || splits.length != card) {
    splits = new InputSplit[card];
  }
  Class<? extends InputSplit>[] cls = new Class[card];
  try {
    for (int i = 0; i < card; ++i) {
      cls[i] =
        Class.forName(Text.readString(in)).asSubclass(InputSplit.class);
    }
    for (int i = 0; i < card; ++i) {
      splits[i] = ReflectionUtils.newInstance(cls[i], null);
      splits[i].readFields(in);
    }
  } catch (ClassNotFoundException e) {
    throw (IOException)new IOException("Failed split init").initCause(e);
  }
}
 
Example #18
Source File: EncodedQualifiersColumnProjectionFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void readFields(DataInput input) throws IOException {
    this.emptyCFName = WritableUtils.readCompressedByteArray(input);
    int bitsetLongArraySize = WritableUtils.readVInt(input);
    long[] bitsetLongArray = new long[bitsetLongArraySize];
    for (int i = 0; i < bitsetLongArraySize; i++) {
        bitsetLongArray[i] = WritableUtils.readVLong(input);
    }
    this.trackedColumns = BitSet.valueOf(bitsetLongArray);
    this.encodingScheme = QualifierEncodingScheme.values()[WritableUtils.readVInt(input)];
    int conditionOnlyCfsSize = WritableUtils.readVInt(input);
    this.conditionOnlyCfs = new TreeSet<byte[]>(Bytes.BYTES_COMPARATOR);
    while (conditionOnlyCfsSize > 0) {
        this.conditionOnlyCfs.add(WritableUtils.readCompressedByteArray(input));
        conditionOnlyCfsSize--;
    }
}
 
Example #19
Source File: TaskStatus.java    From hadoop-gpu with Apache License 2.0 6 votes vote down vote up
public void readFields(DataInput in) throws IOException {
  this.taskid.readFields(in);
  this.progress = in.readFloat();
  this.runState = WritableUtils.readEnum(in, State.class);
  this.diagnosticInfo = Text.readString(in);
  this.stateString = Text.readString(in);
  this.phase = WritableUtils.readEnum(in, Phase.class); 
  this.startTime = in.readLong(); 
  this.finishTime = in.readLong(); 
  counters = new Counters();
  this.includeCounters = in.readBoolean();
  this.outputSize = in.readLong();
  if (includeCounters) {
    counters.readFields(in);
  }
  nextRecordRange.readFields(in);
  this.runOnGPU = in.readBoolean();
}
 
Example #20
Source File: JobStatus.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public synchronized void readFields(DataInput in) throws IOException {
  this.jobid = new JobID();
  this.jobid.readFields(in);
  this.setupProgress = in.readFloat();
  this.mapProgress = in.readFloat();
  this.reduceProgress = in.readFloat();
  this.cleanupProgress = in.readFloat();
  this.runState = WritableUtils.readEnum(in, State.class);
  this.startTime = in.readLong();
  this.user = StringInterner.weakIntern(Text.readString(in));
  this.priority = WritableUtils.readEnum(in, JobPriority.class);
  this.schedulingInfo = StringInterner.weakIntern(Text.readString(in));
  this.finishTime = in.readLong();
  this.isRetired = in.readBoolean();
  this.historyFile = StringInterner.weakIntern(Text.readString(in));
  this.jobName = StringInterner.weakIntern(Text.readString(in));
  this.trackingUrl = StringInterner.weakIntern(Text.readString(in));
  this.jobFile = StringInterner.weakIntern(Text.readString(in));
  this.isUber = in.readBoolean();

  // De-serialize the job's ACLs
  int numACLs = in.readInt();
  for (int i = 0; i < numACLs; i++) {
    JobACL aclType = WritableUtils.readEnum(in, JobACL.class);
    AccessControlList acl = new AccessControlList(" ");
    acl.readFields(in);
    this.jobACLs.put(aclType, acl);
  }
}
 
Example #21
Source File: BlockTokenSecretManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * check if a token is expired. for unit test only. return true when token is
 * expired, false otherwise
 */
static boolean isTokenExpired(Token<BlockTokenIdentifier> token)
    throws IOException {
  ByteArrayInputStream buf = new ByteArrayInputStream(token.getIdentifier());
  DataInputStream in = new DataInputStream(buf);
  long expiryDate = WritableUtils.readVLong(in);
  return isExpired(expiryDate);
}
 
Example #22
Source File: ValueSchema.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
public void write(DataOutput output) throws IOException {
    WritableUtils.writeVInt(output, (type.ordinal() + 1) * (this.isNullable ? -1 : 1));
    WritableUtils.writeVInt(output, count * (columnModifier == null ? 1 : -1));
    if (type.isFixedWidth() && type.getByteSize() == null) {
        WritableUtils.writeVInt(output, byteSize);
    }
}
 
Example #23
Source File: DatabaseDocument.java    From marklogic-contentpump with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
    int ordinal = in.readInt();
    contentType = ContentType.valueOf(ordinal);
    int length = WritableUtils.readVInt(in);
    if (length > MAX_BUFFER_SIZE) {
        is = (DataInputStream)in;
        return;
    }
    content = new byte[length];
    in.readFully(content, 0, length);
}
 
Example #24
Source File: BooleanExpressionFilter.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput input) throws IOException {
    try {
        expression = ExpressionType.values()[WritableUtils.readVInt(input)].newInstance();
        expression.readFields(input);
        expression.reset(); // Initializes expression tree for partial evaluation
    } catch (Throwable t) { // Catches incompatibilities during reading/writing and doesn't retry
        ServerUtil.throwIOException("BooleanExpressionFilter failed during reading", t);
    }
}
 
Example #25
Source File: PrefixByteEncoder.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Prefix encodes the byte array from offset to length into output stream. 
 * Instead of writing the entire byte array, only the portion of the byte array
 * that differs from the beginning of the previous byte array written is written.
 *  
 * @param out output stream to encode into
 * @param b byte array buffer
 * @param offset offset into byte array to start encoding
 * @param length length of byte array to encode
 * @throws IOException
 */
public void encode(DataOutput out, byte[] b, int offset, int length) throws IOException {
      int i = 0;
      int prevOffset = previous.getOffset();
      byte[] prevBytes = previous.get();
      int prevLength = previous.getLength();
      int minLength = prevLength < b.length ? prevLength : b.length;
      for(i = 0; (i < minLength) && (prevBytes[prevOffset + i] == b[offset + i]); i++);
      WritableUtils.writeVInt(out, i);
      Bytes.writeByteArray(out, b, offset + i, length - i);
      previous.set(b, offset, length);
      if (length > maxLength) {
          maxLength = length;
      }
}
 
Example #26
Source File: Bytes.java    From incubator-tajo with Apache License 2.0 5 votes vote down vote up
/**
 * Read byte-array written with a WritableableUtils.vint prefix.
 * @param in Input to read from.
 * @return byte array read off <code>in</code>
 * @throws java.io.IOException e
 */
public static byte [] readByteArray(final DataInput in)
throws IOException {
  int len = WritableUtils.readVInt(in);
  if (len < 0) {
    throw new NegativeArraySizeException(Integer.toString(len));
  }
  byte [] result = new byte[len];
  in.readFully(result, 0, len);
  return result;
}
 
Example #27
Source File: FileSystemCounterGroup.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void readFields(DataInput in) throws IOException {
  int numSchemes = WritableUtils.readVInt(in);    // #scheme
  FileSystemCounter[] enums = FileSystemCounter.values();
  for (int i = 0; i < numSchemes; ++i) {
    String scheme = WritableUtils.readString(in); // scheme
    int numCounters = WritableUtils.readVInt(in); // #counter
    for (int j = 0; j < numCounters; ++j) {
      findCounter(scheme, enums[WritableUtils.readVInt(in)])  // key
          .setValue(WritableUtils.readVLong(in)); // value
    }
  }
}
 
Example #28
Source File: ShuffleHeader.java    From tez with Apache License 2.0 5 votes vote down vote up
public int writeLength() throws IOException {
  int length = 0;
  int mapIdLength = Text.encode(mapId).limit();
  length += mapIdLength;

  length += WritableUtils.getVIntSize(mapIdLength);
  length += WritableUtils.getVIntSize(compressedLength);
  length += WritableUtils.getVIntSize(uncompressedLength);
  length += WritableUtils.getVIntSize(forReduce);

  return length;
}
 
Example #29
Source File: LogBeanWritable.java    From 163-bigdate-note with GNU General Public License v3.0 5 votes vote down vote up
public void readFields(DataInput in) throws IOException {
    activeName = WritableUtils.readString(in);
    sessionID = WritableUtils.readString(in);
    timeTag = in.readLong();
    ip = WritableUtils.readString(in);
    deviceID = WritableUtils.readString(in);
    reqUrl = WritableUtils.readString(in);
    userID = WritableUtils.readString(in);
    productID = WritableUtils.readString(in);
    orderID = WritableUtils.readString(in);
}
 
Example #30
Source File: TupleWritable.java    From RDFS with Apache License 2.0 5 votes vote down vote up
/** Writes each Writable to <code>out</code>.
 * TupleWritable format:
 * {@code
 *  <count><type1><type2>...<typen><obj1><obj2>...<objn>
 * }
 */
public void write(DataOutput out) throws IOException {
  WritableUtils.writeVInt(out, values.length);
  WritableUtils.writeVLong(out, written);
  for (int i = 0; i < values.length; ++i) {
    Text.writeString(out, values[i].getClass().getName());
  }
  for (int i = 0; i < values.length; ++i) {
    if (has(i)) {
      values[i].write(out);
    }
  }
}