Java Code Examples for org.apache.hadoop.io.WritableUtils#decodeVIntSize()

/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

/**
 * Reads a zero-compressed encoded long from input stream and returns it.
 * @param buffer Binary array
 * @param offset Offset into array at which vint begins.
 * @throws java.io.IOException e
 * @return deserialized long from stream.
 */
public static long readVLong(final byte [] buffer, final int offset)
throws IOException {
  byte firstByte = buffer[offset];
  int length = (byte) WritableUtils.decodeVIntSize(firstByte);
  if (length == 1) {
    return firstByte;
  }
  long i = 0;
  for (int idx = 0; idx < length - 1; idx++) {
    byte b = buffer[offset + 1 + idx];
    i = i << 8;
    i = i | (b & 0xFF);
  }
  return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
}
 

/**
 * Decode a vint from the buffer pointed at to by ptr and
 * increment the offset of the ptr by the length of the
 * vint.
 * @param ptr a pointer to a byte array buffer
 * @return the decoded vint value as a long
 */
public static long vlongFromBytes(ImmutableBytesWritable ptr) {
    final byte [] buffer = ptr.get();
    final int offset = ptr.getOffset();
    byte firstByte = buffer[offset];
    int len = WritableUtils.decodeVIntSize(firstByte);
    if (len == 1) {
        ptr.set(buffer, offset+1, ptr.getLength());
        return firstByte;
    }
    long i = 0;
    for (int idx = 0; idx < len-1; idx++) {
        byte b = buffer[offset + 1 + idx];
        i = i << 8;
        i = i | (b & 0xFF);
    }
    ptr.set(buffer, offset+len, ptr.getLength());
    return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

/**
 * @param buffer buffer to convert
 * @return vint bytes as an integer.
 */
public static long bytesToVint(final byte [] buffer) {
  int offset = 0;
  byte firstByte = buffer[offset++];
  int len = WritableUtils.decodeVIntSize(firstByte);
  if (len == 1) {
    return firstByte;
  }
  long i = 0;
  for (int idx = 0; idx < len-1; idx++) {
    byte b = buffer[offset++];
    i = i << 8;
    i = i | (b & 0xFF);
  }
  return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

@Override
public int compare(byte[] b1, int s1, int l1,
                   byte[] b2, int s2, int l2) {
    try {
        int n1 = WritableUtils.decodeVIntSize(b1[s1]);
        int n2 = WritableUtils.decodeVIntSize(b2[s2]);
        IntrinsicsOnly v1 = JsonHelper.fromUtf8Bytes(b1, s1 + n1, l1 - n1, IntrinsicsOnly.class);
        IntrinsicsOnly v2 = JsonHelper.fromUtf8Bytes(b1, s2 + n2, l2 - n2, IntrinsicsOnly.class);

        return ComparisonChain.start()
                .compare(v1.table, v2.table)
                .compare(v1.id, v2.id)
                .compare(v1.version, v2.version)
                .result();
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}
 

public int compare(byte[] b1, int s1, int l1,
    byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();

  if (allKeySpecs.size() == 0) {
    return compareBytes(b1, s1 + n1, l1 - n1, b2, s2 + n2, l2 - n2);
  }
  
  int []lengthIndicesFirst = 
    keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1);
  int []lengthIndicesSecond = 
    keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2);
  
  for (KeyDescription keySpec : allKeySpecs) {
    int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1,
      lengthIndicesFirst, keySpec);
    int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, 
      lengthIndicesFirst, keySpec);
    int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2,
      lengthIndicesSecond, keySpec);
    int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, 
      lengthIndicesSecond, keySpec);
    int result;
    if ((result = compareByteSequence(b1, startCharFirst, endCharFirst, b2, 
        startCharSecond, endCharSecond, keySpec)) != 0) {
      return result;
    }
  }
  return 0;
}
 

public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  n1 -= WritableUtils.getVIntSize(n1);
  n2 -= WritableUtils.getVIntSize(n2);
  return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2);
}
 

public int compare(byte[] b1, int s1, int l1,
    byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();
  if (allKeySpecs.size() == 0) {
    return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2);
  }
  int []lengthIndicesFirst = keyFieldHelper.getWordLengths(b1, s1+n1, s1+l1);
  int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2+n2, s2+l2);
  for (KeyDescription keySpec : allKeySpecs) {
    int startCharFirst = keyFieldHelper.getStartOffset(b1, s1+n1, s1+l1, lengthIndicesFirst,
        keySpec);
    int endCharFirst = keyFieldHelper.getEndOffset(b1, s1+n1, s1+l1, lengthIndicesFirst,
        keySpec);
    int startCharSecond = keyFieldHelper.getStartOffset(b2, s2+n2, s2+l2, lengthIndicesSecond,
        keySpec);
    int endCharSecond = keyFieldHelper.getEndOffset(b2, s2+n2, s2+l2, lengthIndicesSecond,
        keySpec);
    int result;
    if ((result = compareByteSequence(b1, startCharFirst, endCharFirst, b2, 
        startCharSecond, endCharSecond, keySpec)) != 0) {
      return result;
    }
  }
  return 0;
}
 

public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  n1 -= WritableUtils.getVIntSize(n1);
  n2 -= WritableUtils.getVIntSize(n2);
  return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2);
}
 

/**
 * Actually do the mvcc read. Does no checks.
 */
private void _readMvccVersion(int offsetFromPos) {
  // This is Bytes#bytesToVint inlined so can save a few instructions in this hot method; i.e.
  // previous if one-byte vint, we'd redo the vint call to find int size.
  // Also the method is kept small so can be inlined.
  byte firstByte = blockBuffer.getByteAfterPosition(offsetFromPos);
  int len = WritableUtils.decodeVIntSize(firstByte);
  if (len == 1) {
    this.currMemstoreTS = firstByte;
  } else {
    int remaining = len -1;
    long i = 0;
    offsetFromPos++;
    if (remaining >= Bytes.SIZEOF_INT) {
      // The int read has to be converted to unsigned long so the & op
      i = (blockBuffer.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL);
      remaining -= Bytes.SIZEOF_INT;
      offsetFromPos += Bytes.SIZEOF_INT;
    }
    if (remaining >= Bytes.SIZEOF_SHORT) {
      short s = blockBuffer.getShortAfterPosition(offsetFromPos);
      i = i << 16;
      i = i | (s & 0xFFFF);
      remaining -= Bytes.SIZEOF_SHORT;
      offsetFromPos += Bytes.SIZEOF_SHORT;
    }
    for (int idx = 0; idx < remaining; idx++) {
      byte b = blockBuffer.getByteAfterPosition(offsetFromPos + idx);
      i = i << 8;
      i = i | (b & 0xFF);
    }
    currMemstoreTS = (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
  }
  this.currMemstoreTSLen = len;
}
 

private static long readVLong(ByteVisitor visitor) {
  byte firstByte = visitor.get();
  int len = WritableUtils.decodeVIntSize(firstByte);
  if (len == 1) {
    return firstByte;
  }
  long i = 0;
  for (int idx = 0; idx < len - 1; idx++) {
    byte b = visitor.get();
    i = i << 8;
    i = i | (b & 0xFF);
  }
  return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
}
 

public int compare(byte[] b1, int s1, int l1,
    byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();

  if (allKeySpecs.size() == 0) {
    return compareBytes(b1, s1 + n1, l1 - n1, b2, s2 + n2, l2 - n2);
  }
  
  int []lengthIndicesFirst = 
    keyFieldHelper.getWordLengths(b1, s1 + n1, s1 + l1);
  int []lengthIndicesSecond = 
    keyFieldHelper.getWordLengths(b2, s2 + n2, s2 + l2);
  
  for (KeyDescription keySpec : allKeySpecs) {
    int startCharFirst = keyFieldHelper.getStartOffset(b1, s1 + n1, s1 + l1,
      lengthIndicesFirst, keySpec);
    int endCharFirst = keyFieldHelper.getEndOffset(b1, s1 + n1, s1 + l1, 
      lengthIndicesFirst, keySpec);
    int startCharSecond = keyFieldHelper.getStartOffset(b2, s2 + n2, s2 + l2,
      lengthIndicesSecond, keySpec);
    int endCharSecond = keyFieldHelper.getEndOffset(b2, s2 + n2, s2 + l2, 
      lengthIndicesSecond, keySpec);
    int result;
    if ((result = compareByteSequence(b1, startCharFirst, endCharFirst, b2, 
        startCharSecond, endCharSecond, keySpec)) != 0) {
      return result;
    }
  }
  return 0;
}
 

/**
 * Reads a zero-compressed encoded long from input buffer and returns it.
 * @param buffer Binary array
 * @param offset Offset into array at which vint begins.
 * @return deserialized long from buffer.
 */
public static long readAsVLong(final byte [] buffer, final int offset) {
  byte firstByte = buffer[offset];
  int len = WritableUtils.decodeVIntSize(firstByte);
  if (len == 1) {
    return firstByte;
  }
  long i = 0;
  for (int idx = 0; idx < len-1; idx++) {
    byte b = buffer[offset + 1 + idx];
    i = i << 8;
    i = i | (b & 0xFF);
  }
  return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i);
}
 

public int compare(byte[] b1, int s1, int l1,
    byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  List <KeyDescription> allKeySpecs = keyFieldHelper.keySpecs();
  if (allKeySpecs.size() == 0) {
    return compareBytes(b1, s1+n1, l1-n1, b2, s2+n2, l2-n2);
  }
  int []lengthIndicesFirst = keyFieldHelper.getWordLengths(b1, s1+n1, s1+l1);
  int []lengthIndicesSecond = keyFieldHelper.getWordLengths(b2, s2+n2, s2+l2);
  for (KeyDescription keySpec : allKeySpecs) {
    int startCharFirst = keyFieldHelper.getStartOffset(b1, s1+n1, s1+l1, lengthIndicesFirst,
        keySpec);
    int endCharFirst = keyFieldHelper.getEndOffset(b1, s1+n1, s1+l1, lengthIndicesFirst,
        keySpec);
    int startCharSecond = keyFieldHelper.getStartOffset(b2, s2+n2, s2+l2, lengthIndicesSecond,
        keySpec);
    int endCharSecond = keyFieldHelper.getEndOffset(b2, s2+n2, s2+l2, lengthIndicesSecond,
        keySpec);
    int result;
    if ((result = compareByteSequence(b1, startCharFirst, endCharFirst, b2, 
        startCharSecond, endCharSecond, keySpec)) != 0) {
      return result;
    }
  }
  return 0;
}
 

public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  return -1 * WritableComparator.compareBytes(b1, s1+n1, l1-n1,
                                              b2, s2+n2, l2-n2);
}
 

/** {@inheritDoc} */
@Override public int compare(Text val1, long val2Ptr, int val2Len) {
    int len2 = WritableUtils.decodeVIntSize(GridUnsafe.getByte(val2Ptr));

    return HadoopUtils.compareBytes(val1.getBytes(), val1.getLength(), val2Ptr + len2, val2Len - len2);
}
 

private HashCacheImpl(byte[] hashCacheBytes, MemoryChunk memoryChunk) {
    try {
        this.memoryChunk = memoryChunk;
        byte[] hashCacheByteArray = hashCacheBytes;
        int offset = 0;
        ByteArrayInputStream input = new ByteArrayInputStream(hashCacheByteArray, offset, hashCacheBytes.length);
        DataInputStream dataInput = new DataInputStream(input);
        int nExprs = dataInput.readInt();
        List<Expression> onExpressions = new ArrayList<Expression>(nExprs);
        for (int i = 0; i < nExprs; i++) {
            int expressionOrdinal = WritableUtils.readVInt(dataInput);
            Expression expression = ExpressionType.values()[expressionOrdinal].newInstance();
            expression.readFields(dataInput);
            onExpressions.add(expression);                        
        }
        boolean singleValueOnly = false;
        int exprSizeAndSingleValueOnly = dataInput.readInt();
        int exprSize = exprSizeAndSingleValueOnly;
        if (exprSize < 0) {
            exprSize *= -1;
            singleValueOnly = true;
        }
        this.singleValueOnly = singleValueOnly;
        offset += exprSize;
        int nRows = dataInput.readInt();
        long estimatedSize = SizedUtil.sizeOfMap(nRows, SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE, SizedUtil.RESULT_SIZE) + hashCacheBytes.length;
        this.memoryChunk.resize(estimatedSize);
        HashMap<ImmutableBytesPtr,List<Tuple>> hashCacheMap = new HashMap<ImmutableBytesPtr,List<Tuple>>(nRows * 5 / 4);
        offset += Bytes.SIZEOF_INT;
        // Build Map with evaluated hash key as key and row as value
        for (int i = 0; i < nRows; i++) {
            int resultSize = (int)Bytes.readVLong(hashCacheByteArray, offset);
            offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]);
            ImmutableBytesWritable value = new ImmutableBytesWritable(hashCacheByteArray,offset,resultSize);
            Tuple result = new ResultTuple(ResultUtil.toResult(value));
            ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, onExpressions);
            List<Tuple> tuples = hashCacheMap.get(key);
            if (tuples == null) {
                tuples = new LinkedList<Tuple>();
                hashCacheMap.put(key, tuples);
            }
            tuples.add(result);
            offset += resultSize;
        }
        this.hashCache = Collections.unmodifiableMap(hashCacheMap);
    } catch (IOException e) { // Not possible with ByteArrayInputStream
        throw new RuntimeException(e);
    }
}
 

private HashCacheImpl(byte[] hashCacheBytes, MemoryChunk memoryChunk) {
    try {
        this.memoryChunk = memoryChunk;
        byte[] hashCacheByteArray = hashCacheBytes;
        int offset = 0;
        ByteArrayInputStream input = new ByteArrayInputStream(hashCacheByteArray, offset, hashCacheBytes.length);
        DataInputStream dataInput = new DataInputStream(input);
        int nExprs = dataInput.readInt();
        List<Expression> onExpressions = new ArrayList<Expression>(nExprs);
        for (int i = 0; i < nExprs; i++) {
            int expressionOrdinal = WritableUtils.readVInt(dataInput);
            Expression expression = ExpressionType.values()[expressionOrdinal].newInstance();
            expression.readFields(dataInput);
            onExpressions.add(expression);                        
        }
        int exprSize = dataInput.readInt();
        offset += exprSize;
        int nRows = dataInput.readInt();
        int estimatedSize = SizedUtil.sizeOfMap(nRows, SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE, SizedUtil.RESULT_SIZE) + hashCacheBytes.length;
        this.memoryChunk.resize(estimatedSize);
        HashMap<ImmutableBytesPtr,List<Tuple>> hashCacheMap = new HashMap<ImmutableBytesPtr,List<Tuple>>(nRows * 5 / 4);
        offset += Bytes.SIZEOF_INT;
        // Build Map with evaluated hash key as key and row as value
        for (int i = 0; i < nRows; i++) {
            int resultSize = (int)Bytes.readVLong(hashCacheByteArray, offset);
            offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]);
            ImmutableBytesWritable value = new ImmutableBytesWritable(hashCacheByteArray,offset,resultSize);
            Tuple result = new ResultTuple(new Result(value));
            ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, onExpressions);
            List<Tuple> tuples = hashCacheMap.get(key);
            if (tuples == null) {
                tuples = new ArrayList<Tuple>(1);
                hashCacheMap.put(key, tuples);
            }
            tuples.add(result);
            offset += resultSize;
        }
        this.hashCache = Collections.unmodifiableMap(hashCacheMap);
    } catch (IOException e) { // Not possible with ByteArrayInputStream
        throw new RuntimeException(e);
    }
}
 

private HashCacheImpl(byte[] hashCacheBytes, MemoryChunk memoryChunk, int clientVersion) {
    try {
        this.memoryChunk = memoryChunk;
        this.clientVersion = clientVersion;
        byte[] hashCacheByteArray = hashCacheBytes;
        int offset = 0;
        ByteArrayInputStream input = new ByteArrayInputStream(hashCacheByteArray, offset, hashCacheBytes.length);
        DataInputStream dataInput = new DataInputStream(input);
        int nExprs = dataInput.readInt();
        List<Expression> onExpressions = new ArrayList<Expression>(nExprs);
        for (int i = 0; i < nExprs; i++) {
            int expressionOrdinal = WritableUtils.readVInt(dataInput);
            Expression expression = ExpressionType.values()[expressionOrdinal].newInstance();
            expression.readFields(dataInput);
            onExpressions.add(expression);                        
        }
        boolean singleValueOnly = false;
        int exprSizeAndSingleValueOnly = dataInput.readInt();
        int exprSize = exprSizeAndSingleValueOnly;
        if (exprSize < 0) {
            exprSize *= -1;
            singleValueOnly = true;
        }
        this.singleValueOnly = singleValueOnly;
        offset += exprSize;
        int nRows = dataInput.readInt();
        long estimatedSize = SizedUtil.sizeOfMap(nRows, SizedUtil.IMMUTABLE_BYTES_WRITABLE_SIZE, SizedUtil.RESULT_SIZE) + hashCacheBytes.length;
        this.memoryChunk.resize(estimatedSize);
        HashMap<ImmutableBytesPtr,List<Tuple>> hashCacheMap = new HashMap<ImmutableBytesPtr,List<Tuple>>(nRows * 5 / 4);
        offset += Bytes.SIZEOF_INT;
        // Build Map with evaluated hash key as key and row as value
        for (int i = 0; i < nRows; i++) {
            int resultSize = (int)Bytes.readVLong(hashCacheByteArray, offset);
            offset += WritableUtils.decodeVIntSize(hashCacheByteArray[offset]);
            ImmutableBytesWritable value = new ImmutableBytesWritable(hashCacheByteArray,offset,resultSize);
            Tuple result = new ResultTuple(ResultUtil.toResult(value));
            ImmutableBytesPtr key = TupleUtil.getConcatenatedValue(result, onExpressions);
            List<Tuple> tuples = hashCacheMap.get(key);
            if (tuples == null) {
                tuples = new LinkedList<Tuple>();
                hashCacheMap.put(key, tuples);
            }
            tuples.add(result);
            offset += resultSize;
        }
        this.hashCache = Collections.unmodifiableMap(hashCacheMap);
    } catch (IOException e) { // Not possible with ByteArrayInputStream
        throw new RuntimeException(e);
    }
}
 

public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
  int n1 = WritableUtils.decodeVIntSize(b1[s1]);
  int n2 = WritableUtils.decodeVIntSize(b2[s2]);
  return -1 * WritableComparator.compareBytes(b1, s1+n1, l1-n1,
                                              b2, s2+n2, l2-n2);
}