Java Code Examples for org.apache.hadoop.hbase.io.ImmutableBytesWritable#getOffset()

@Test
public void testForVarCharArrayForOddNumberWithIndex6() {
    String[] strArr = new String[6];
    strArr[0] = "abx";
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = null;
    strArr[4] = "random12";
    strArr[5] = "random17";
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("random12", Bytes.toString(res));
}
 

public void or(ImmutableBytesWritable ptr) {
    if (schema == null) {
        return;
    }
    if (isVarLength()) {
        int offset = ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_SHORT;
        short nLongs = Bytes.toShort(ptr.get(), offset);
        offset -= nLongs * Bytes.SIZEOF_LONG;
        for (int i = 0; i < nLongs; i++) {
            bits[i] |= Bytes.toLong(ptr.get(), offset);
            offset += Bytes.SIZEOF_LONG;
        }
        maxSetBit = Math.max(maxSetBit, nLongs * Bytes.SIZEOF_LONG - 1);
    } else {
        long l = Bytes.toShort(ptr.get(), ptr.getOffset() + ptr.getLength() - Bytes.SIZEOF_SHORT);
        bits[0] |= l;
        maxSetBit = Math.max(maxSetBit, BITS_PER_SHORT - 1);
    }
    
}
 

@Test
public void testPositionSearchWithVarLengthArrayWithNullValue5() {
    String[] strArr = new String[5];
    strArr[0] = "abx";
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = null;
    strArr[4] = "ran";
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("", Bytes.toString(res));
}
 

@Test
public void testLongArrayWithIndex() {
	Long[] longArr = new Long[4];
	longArr[0] = 1l;
	longArr[1] = 2l;
	longArr[2] = 4l;
	longArr[3] = 5l;
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PLong.INSTANCE, longArr);
	PLongArray.INSTANCE.toObject(arr, PLongArray.INSTANCE);
	byte[] bytes = PLongArray.INSTANCE.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
	PArrayDataType.positionAtArrayElement(ptr, 2, PLong.INSTANCE, PLong.INSTANCE.getByteSize());
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	long result = (Long) PLong.INSTANCE.toObject(res);
	assertEquals(4l, result);
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
       try {
           KeyValueSchema schema = getSchema();
           TupleProjector.decodeProjectedValue(tuple, ptr);
           bitSet.clear();
           bitSet.or(ptr);
           int maxOffset = ptr.getOffset() + ptr.getLength() - bitSet.getEstimatedLength();
           schema.iterator(ptr, position, bitSet);
           Boolean hasValue = schema.next(ptr, position, maxOffset, bitSet);
           if (hasValue == null || !hasValue.booleanValue()) {
               ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
               return true;
           }
       } catch (IOException e) {
           return false;
       }
	
	return true;
}
 

@Test
public void testLongArrayWithIndex() {
	Long[] longArr = new Long[4];
	longArr[0] = 1l;
	longArr[1] = 2l;
	longArr[2] = 4l;
	longArr[3] = 5l;
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PDataType.LONG, longArr);
	PDataType.LONG_ARRAY.toObject(arr, PDataType.LONG_ARRAY);
	byte[] bytes = PDataType.LONG_ARRAY.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
	PArrayDataType.positionAtArrayElement(ptr, 2, PDataType.LONG);
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	long result = (Long) PDataType.LONG.toObject(res);
	assertEquals(4l, result);
}
 

public static ProjectedValueTuple mergeProjectedValue(ProjectedValueTuple dest, KeyValueSchema destSchema, ValueBitSet destBitSet,
		Tuple src, KeyValueSchema srcSchema, ValueBitSet srcBitSet, int offset) throws IOException {
	ImmutableBytesWritable destValue = new ImmutableBytesWritable(dest.getProjectedValue());
	destBitSet.clear();
	destBitSet.or(destValue);
	int origDestBitSetLen = dest.getBitSetLength();
	ImmutableBytesWritable srcValue = new ImmutableBytesWritable();
	decodeProjectedValue(src, srcValue);
	srcBitSet.clear();
	srcBitSet.or(srcValue);
	int origSrcBitSetLen = srcBitSet.getEstimatedLength();
	for (int i = 0; i < srcBitSet.getMaxSetBit(); i++) {
		if (srcBitSet.get(i)) {
			destBitSet.set(offset + i);
		}
	}
	int destBitSetLen = destBitSet.getEstimatedLength();
	byte[] merged = new byte[destValue.getLength() - origDestBitSetLen + srcValue.getLength() - origSrcBitSetLen + destBitSetLen];
	int o = Bytes.putBytes(merged, 0, destValue.get(), destValue.getOffset(), destValue.getLength() - origDestBitSetLen);
	o = Bytes.putBytes(merged, o, srcValue.get(), srcValue.getOffset(), srcValue.getLength() - origSrcBitSetLen);
	destBitSet.toBytes(merged, o);
	ImmutableBytesWritable keyPtr = dest.getKeyPtr();
    return new ProjectedValueTuple(keyPtr.get(), keyPtr.getOffset(), keyPtr.getLength(), dest.getTimestamp(), merged, destBitSetLen);
}
 

@Test
public void testPositionSearchWithVarLengthArrayWithNullValueAtTheStart1() {
    String[] strArr = new String[5];
    strArr[0] = null;
    strArr[1] = "ereref";
    strArr[2] = "random";
    strArr[3] = null;
    strArr[4] = "ran";
    PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
            PVarchar.INSTANCE, strArr);
    byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
    ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
    PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    byte[] bs = ptr.get();
    byte[] res = new byte[length];
    System.arraycopy(bs, offset, res, 0, length);
    assertEquals("", Bytes.toString(res));
}
 

@Test
public void testForVarCharArrayForOddNumberWithIndex() {
	String[] strArr = new String[5];
	strArr[0] = "abx";
	strArr[1] = "ereref";
	strArr[2] = "random";
	strArr[3] = "random12";
	strArr[4] = "ran";
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PVarchar.INSTANCE, strArr);
	byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
	PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	assertEquals("random12", Bytes.toString(res));
}
 

@Test
public void testForVarCharArrayForOneElementArrayWithIndex() {
	String[] strArr = new String[1];
	strArr[0] = "abx";
	PhoenixArray arr = PArrayDataType.instantiatePhoenixArray(
			PVarchar.INSTANCE, strArr);
	byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
	ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
	PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
	int offset = ptr.getOffset();
	int length = ptr.getLength();
	byte[] bs = ptr.get();
	byte[] res = new byte[length];
	System.arraycopy(bs, offset, res, 0, length);
	assertEquals("abx", Bytes.toString(res));
}
 

/**
 * Move the bytes ptr to the next position relative to the current ptr
 * @param ptr bytes pointer pointing to the value at the positional index
 * provided.
 * @param position zero-based index of the next field in the value schema
 * @param maxOffset max possible offset value when iterating
 * @return true if a value was found and ptr was set, false if the value is null and ptr was not
 * set, and null if the value is null and there are no more values
  */
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value="NP_BOOLEAN_RETURN_NULL", 
        justification="Designed to return null.")
public Boolean next(ImmutableBytesWritable ptr, int position, int maxOffset, ValueBitSet valueSet) {
    if (ptr.getOffset() + ptr.getLength() >= maxOffset) {
        ptr.set(ptr.get(), maxOffset, 0);
        return null;
    }
    if (position >= getFieldCount()) {
        return null;
    }
    // Move the pointer past the current value and set length
    // to 0 to ensure you never set the ptr past the end of the
    // backing byte array.
    ptr.set(ptr.get(), ptr.getOffset() + ptr.getLength(), 0);
    if (!isNull(position, valueSet)) {
        Field field = this.getField(position);
        int length = field.getDataType().isFixedWidth() ? 
                field.getByteSize() : ByteUtil.vintFromBytes(ptr);
        if (ptr.getOffset() + length > maxOffset) {
            throw new RuntimeException(new SQLExceptionInfo.Builder(SQLExceptionCode.ILLEGAL_DATA)
                .setMessage("Expected length of at least " + length + " bytes, but had " + (maxOffset
                                - ptr.getOffset())).build().buildException());
        }
        ptr.set(ptr.get(),ptr.getOffset(),length);
        return ptr.getLength() > 0;
    }
    return false;
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {

    if (!getLHSExpr().evaluate(tuple, ptr)){
        return false;
    }
    boolean isLHSRowKeyOrderOptimized = PArrayDataType.isRowKeyOrderOptimized(getLHSExpr().getDataType(), getLHSExpr().getSortOrder(), ptr);

    SortOrder sortOrder = getRHSExpr().getSortOrder();
    int actualLengthOfArray1 = Math.abs(PArrayDataType.getArrayLength(ptr, getLHSBaseType(), getLHSExpr().getMaxLength()));
    int lengthArray1 = ptr.getLength();
    int offsetArray1 = ptr.getOffset();
    byte[] array1Bytes = ptr.get();
    if (!getRHSExpr().evaluate(tuple, ptr)) {
        return false;
    }
    // If second array is null, return first array
    if (ptr.getLength() == 0){
        ptr.set(array1Bytes, offsetArray1, lengthArray1);
        return true;
    }

    checkSizeCompatibility(ptr, sortOrder, getLHSExpr(), getLHSExpr().getDataType(), getRHSExpr(),getRHSExpr().getDataType());

    // FIXME: calling version of coerceBytes that takes into account the separator used by LHS
    // If the RHS does not have the same separator, it'll be coerced to use it. It's unclear
    // if we should do the same for all classes derived from the base class.
    // Coerce RHS to LHS type
    getLHSExpr().getDataType().coerceBytes(ptr, null, getRHSExpr().getDataType(), getRHSExpr().getMaxLength(),
            getRHSExpr().getScale(), getRHSExpr().getSortOrder(), getLHSExpr().getMaxLength(),
            getLHSExpr().getScale(), getLHSExpr().getSortOrder(), isLHSRowKeyOrderOptimized);
    if (lengthArray1 == 0) {
        return true;
    }
    return modifierFunction(ptr, lengthArray1, offsetArray1, array1Bytes, getLHSBaseType(), actualLengthOfArray1, getMaxLength(), getLHSExpr());
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    Expression child = children.get(0);
    if (!child.evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) {
        return true;
    }
    byte serializedByte = ptr.get()[ptr.getOffset()];
    PIndexState indexState = PIndexState.fromSerializedValue(serializedByte);
    ptr.set(indexState.toBytes());
    return true;
}
 

/**
 * Move the bytes ptr to the next position relative to the current ptr
 * @param ptr bytes pointer pointing to the value at the positional index
 * provided.
 * @param position zero-based index of the next field in the value schema
 * @param maxOffset max possible offset value when iterating
 * @return true if a value was found and ptr was set, false if the value is null and ptr was not
 * set, and null if the value is null and there are no more values
  */
@edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value="NP_BOOLEAN_RETURN_NULL", 
        justification="Designed to return null.")
public Boolean next(ImmutableBytesWritable ptr, int position, int maxOffset, ValueBitSet valueSet) {
    if (ptr.getOffset() + ptr.getLength() >= maxOffset) {
        ptr.set(ptr.get(), maxOffset, 0);
        return null;
    }
    if (position >= getFieldCount()) {
        return null;
    }
    // Move the pointer past the current value and set length
    // to 0 to ensure you never set the ptr past the end of the
    // backing byte array.
    ptr.set(ptr.get(), ptr.getOffset() + ptr.getLength(), 0);
    if (!isNull(position, valueSet)) {
        Field field = this.getField(position);
        if (field.getDataType().isFixedWidth()) {
            ptr.set(ptr.get(),ptr.getOffset(), field.getByteSize());
        } else {
            int length = ByteUtil.vintFromBytes(ptr);
            ptr.set(ptr.get(),ptr.getOffset(),length);
        }
        return ptr.getLength() > 0;
    }
    return false;
}
 

@Override
public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
    Expression child = children.get(0);
    if (!child.evaluate(tuple, ptr)) {
        return false;
    }
    if (ptr.getLength() == 0) {
        return true;
    }
    byte serializedByte = ptr.get()[ptr.getOffset()];
    PIndexState indexState = PIndexState.fromSerializedValue(serializedByte);
    ptr.set(indexState.toBytes());
    return true;
}
 

@Override
public void aggregate(Tuple tuple, ImmutableBytesWritable ptr) {
    ImmutableBytesPtr key = new ImmutableBytesPtr(ptr.get(), ptr.getOffset(), ptr.getLength());
    Integer count = this.valueVsCount.get(key);
    if (count == null) {
        this.valueVsCount.put(key, 1);
    } else {
        this.valueVsCount.put(key, ++count);
    }
}
 

public static boolean equals(Tuple t1, Tuple t2, ImmutableBytesWritable ptr) {
    t1.getKey(ptr);
    byte[] buf = ptr.get();
    int offset = ptr.getOffset();
    int length = ptr.getLength();
    t2.getKey(ptr);
    return Bytes.compareTo(buf, offset, length, ptr.get(), ptr.getOffset(), ptr.getLength()) == 0;
}
 

/**
 * Separate the actual cell data from the serialized list of dynamic column PColumns and
 * return the deserialized list of dynamic column PColumns for the current row
 * @return Deserialized list of dynamic column PColumns or null if there are no dynamic columns
 */
private List<PColumn> getDynColsListAndSeparateFromActualData() {
    Cell base = this.currentRow.getValue(0);
    final byte[] valueArray = CellUtil.cloneValue(base);
    // We inserted the known byte array before appending the serialized list of dynamic columns
    final byte[] anchor = Arrays.copyOf(DYN_COLS_METADATA_CELL_QUALIFIER,
            DYN_COLS_METADATA_CELL_QUALIFIER.length);
    // Reverse the arrays to find the last occurrence of the sub-array in the value array
    ArrayUtils.reverse(valueArray);
    ArrayUtils.reverse(anchor);
    final int pos = valueArray.length - Bytes.indexOf(valueArray, anchor);
    // There are no dynamic columns to process so return immediately
    if (pos >= valueArray.length) {
        return null;
    }
    ArrayUtils.reverse(valueArray);

    // Separate the serialized list of dynamic column PColumns from the actual cell data
    byte[] actualCellDataBytes = Arrays.copyOfRange(valueArray, 0,
            pos - DYN_COLS_METADATA_CELL_QUALIFIER.length);
    ImmutableBytesWritable actualCellData = new ImmutableBytesWritable(actualCellDataBytes);
    ImmutableBytesWritable key = new ImmutableBytesWritable();
    currentRow.getKey(key);
    // Store only the actual cell data as part of the current row
    this.currentRow = new TupleProjector.ProjectedValueTuple(key.get(), key.getOffset(),
            key.getLength(), base.getTimestamp(),
            actualCellData.get(), actualCellData.getOffset(), actualCellData.getLength(), 0);

    byte[] dynColsListBytes = Arrays.copyOfRange(valueArray, pos, valueArray.length);
    List<PColumn> dynCols = new ArrayList<>();
    try {
        List<PTableProtos.PColumn> dynColsProtos = DynamicColumnMetaDataProtos
                .DynamicColumnMetaData.parseFrom(dynColsListBytes).getDynamicColumnsList();
        for (PTableProtos.PColumn colProto : dynColsProtos) {
            dynCols.add(PColumnImpl.createFromProto(colProto));
        }
    } catch (InvalidProtocolBufferException e) {
        return null;
    }
    return dynCols;
}
 

public static void positionAtArrayElement(ImmutableBytesWritable ptr, int arrayIndex, PDataType baseDataType,
    Integer byteSize) {
    byte[] bytes = ptr.get();
    int initPos = ptr.getOffset();
    if (!baseDataType.isFixedWidth()) {
        int noOfElements = Bytes.toInt(bytes, (ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + Bytes.SIZEOF_INT)),
                Bytes.SIZEOF_INT);
        boolean useShort = true;
        if (noOfElements < 0) {
            noOfElements = -noOfElements;
            useShort = false;
        }
        if (arrayIndex >= noOfElements) {
            ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
            return;
        }

        int indexOffset = Bytes.toInt(bytes,
                (ptr.getOffset() + ptr.getLength() - (Bytes.SIZEOF_BYTE + 2 * Bytes.SIZEOF_INT))) + ptr.getOffset();
        if(arrayIndex >= noOfElements) {
            ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        } else {
            // Skip those many offsets as given in the arrayIndex
            // If suppose there are 5 elements in the array and the arrayIndex = 3
            // This means we need to read the 4th element of the array
            // So inorder to know the length of the 4th element we will read the offset of 4th element and the
            // offset of 5th element.
            // Subtracting the offset of 5th element and 4th element will give the length of 4th element
            // So we could just skip reading the other elements.
            int currOffset = getOffset(bytes, arrayIndex, useShort, indexOffset);
            int elementLength = 0;
            if (arrayIndex == (noOfElements - 1)) {
                elementLength = bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE ? 0 : indexOffset
                        - (currOffset + initPos) - 3;
            } else {
                elementLength = bytes[currOffset + initPos] == QueryConstants.SEPARATOR_BYTE ? 0 : getOffset(bytes,
                        arrayIndex + 1, useShort, indexOffset) - currOffset - 1;
            }
            ptr.set(bytes, currOffset + initPos, elementLength);
        }
    } else {
        int elemByteSize = (byteSize == null ? baseDataType.getByteSize() : byteSize);
        int offset = arrayIndex * elemByteSize;
        if (offset >= ptr.getLength()) {
            ptr.set(ByteUtil.EMPTY_BYTE_ARRAY);
        } else {
            ptr.set(bytes, ptr.getOffset() + offset, elemByteSize);
        }
    }
}
 

@edu.umd.cs.findbugs.annotations.SuppressWarnings(
        value="NP_BOOLEAN_RETURN_NULL", 
        justification="Designed to return null.")
public Boolean previous(ImmutableBytesWritable ptr, int position, int minOffset) {
    if (position < 0) {
        return null;
    }
    Field field = this.getField(position);
    if (field.getDataType().isFixedWidth()) {
        ptr.set(ptr.get(), ptr.getOffset()-field.getByteSize(), field.getByteSize());
        return true;
    }
    // If ptr has length of zero, it is assumed that we're at the end of the row key
    int offsetAdjustment = position + 1 == this.getFieldCount() || ptr.getLength() == 0 ? 0 : 1;
    if (position == 0) {
        ptr.set(ptr.get(), minOffset, ptr.getOffset() - minOffset - offsetAdjustment);
        return true;
    }
    field = this.getField(position-1);
    // Field before the one we want to position at is variable length
    // In this case, we can search backwards for our separator byte
    // to determine the length
    if (!field.getDataType().isFixedWidth()) {
        byte[] buf = ptr.get();
        int offset = ptr.getOffset()-1-offsetAdjustment;
        while (offset > minOffset /* sanity check*/ && buf[offset] != QueryConstants.SEPARATOR_BYTE) {
            offset--;
        }
        if (offset == minOffset) { // shouldn't happen
            ptr.set(buf, minOffset, ptr.getOffset()-minOffset-1);
        } else {
            ptr.set(buf,offset+1,ptr.getOffset()-1-offsetAdjustment-offset); // Don't include null terminator in length
        }
        return true;
    }
    int i,fixedOffset = field.getByteSize();
    for (i = position-2; i >= 0 && this.getField(i).getDataType().isFixedWidth(); i--) {
        fixedOffset += this.getField(i).getByteSize();
    }
    // All of the previous fields are fixed width, so we can calculate the offset
    // based on the total fixed offset
    if (i < 0) {
        int length = ptr.getOffset() - fixedOffset - minOffset - offsetAdjustment;
        ptr.set(ptr.get(),minOffset+fixedOffset, length);
        return true;
    }
    // Otherwise we're stuck with starting from the minOffset and working all the way forward,
    // because we can't infer the length of the previous position.
    return iterator(ptr.get(), minOffset, ptr.getOffset() - minOffset - offsetAdjustment, ptr, position+1);
}