org.apache.arrow.vector.BitVector Java Examples
The following examples show how to use
org.apache.arrow.vector.BitVector.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
| Source File: TestData.java From dremio-oss with Apache License 2.0 | 6 votes |
|
private static Pair<BitVector, ResultVerifier> testBitVector(final int startIndexInCurrentOutput, final int startIndexInJob) {
BitVector colBitV = new BitVector("colBit", allocator);
colBitV.allocateNew(5);
colBitV.set(0, 1);
colBitV.set(1, 0);
colBitV.setNull(2);
colBitV.set(3, 1);
colBitV.set(4, 1);
ResultVerifier verifier = new ResultVerifier() {
@Override
public void verify(DataPOJO output) {
int index = startIndexInCurrentOutput;
assertTrue((Boolean)output.extractValue("colBit", index++));
assertFalse((Boolean)output.extractValue("colBit", index++));
assertNull(output.extractValue("colBit", index++));
assertTrue((Boolean)output.extractValue("colBit", index++));
assertTrue((Boolean)output.extractValue("colBit", index));
}
};
return Pair.of(colBitV, verifier);
}
Example #2
| Source File: TestBackwardsCompatibilityHandler.java From dremio-oss with Apache License 2.0 | 6 votes |
|
@Test
public void testConvertBitsToBytes() {
try (BitVector bits = new BitVector("$bits$", allocator);
UInt1Vector bytes = new UInt1Vector("$bits$", allocator);
) {
int count = 100;
for (int i = 0; i < count ; i++) {
bits.setSafe(i, i % 2);
}
bits.setValueCount(count);
ArrowBuf oldBuf = bits.getDataBuffer();
oldBuf.retain();
SerializedField.Builder fieldBuilder = TypeHelper.getMetadataBuilder(bits);
ArrowBuf newBuf = convertBitsToBytes(allocator, fieldBuilder, oldBuf.asNettyBuffer()).arrowBuf();
bytes.setValueCount(count);
SerializedField.Builder newfieldBuilder = TypeHelper.getMetadataBuilder(bytes);
TypeHelper.loadData(bytes, newfieldBuilder.build(), newBuf);
for (int i = 0; i < count ; i++) {
assertEquals(i % 2, bytes.get(i));
}
newBuf.release();
}
}
Example #3
| Source File: TestArrowFileReader.java From dremio-oss with Apache License 2.0 | 6 votes |
|
/** Helper method to get the values in given range in colBit vector used in this test class. */
private static List<Boolean> getBitValues(VectorContainer container, int start, int end) {
FieldReader reader = container.getValueAccessorById(BitVector.class, 0).getValueVector().getReader();
List<Boolean> values = Lists.newArrayList();
for(int i=start; i<end; i++) {
reader.setPosition(i);
if (reader.isSet()) {
values.add(reader.readBoolean());
} else {
values.add(null);
}
}
return values;
}
Example #4
| Source File: TestBoundedPivots.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private static Boolean[] populateBooleanValues(BitVector vector, int size) {
assert size >= 4096;
vector.allocateNew(size);
Boolean[] booleanValues = new Boolean[size];
for (int i = 0; i < size; i++) {
if (i < 64 || (i >= 256 && i < 256 + 64)) {
vector.setNull(i);
continue;
}
if ((i % 6) != 0) {
/* every 6th value in boolean column is null */
if ((i & 1) == 0) {
/* column value true */
vector.set(i, 1);
booleanValues[i] = true;
} else {
/* column value false */
vector.set(i, 0);
booleanValues[i] = false;
}
} else {
vector.setNull(i);
booleanValues[i] = null;
}
}
vector.setValueCount(size);
return booleanValues;
}
Example #5
| Source File: RowBooleanWriter.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void doWrite(Row value, int ordinal) {
if (value.getField(ordinal) == null) {
((BitVector) getValueVector()).setNull(getCount());
} else if ((boolean) value.getField(ordinal)) {
((BitVector) getValueVector()).setSafe(getCount(), 1);
} else {
((BitVector) getValueVector()).setSafe(getCount(), 0);
}
}
Example #6
| Source File: BooleanWriter.java From flink with Apache License 2.0 | 5 votes |
|
@Override
public void doWrite(T in, int ordinal) {
if (isNullAt(in, ordinal)) {
((BitVector) getValueVector()).setNull(getCount());
} else if (readBoolean(in, ordinal)) {
((BitVector) getValueVector()).setSafe(getCount(), 1);
} else {
((BitVector) getValueVector()).setSafe(getCount(), 0);
}
}
Example #7
| Source File: TestArrowFileReader.java From dremio-oss with Apache License 2.0 | 5 votes |
|
/** Helper method which creates a test bit vector */
private static BitVector testBitVector(BufferAllocator allocator) {
BitVector colBitV = new BitVector("colBit", allocator);
colBitV.allocateNew(5);
for(int i=0; i<TEST_BIT_VALUES.size(); i++) {
if (TEST_BIT_VALUES.get(i) == null) {
colBitV.setNull(i);
} else {
colBitV.set(i, TEST_BIT_VALUES.get(i) ? 1 : 0);
}
}
return colBitV;
}
Example #8
| Source File: TestBoundedPivots.java From dremio-oss with Apache License 2.0 | 5 votes |
|
@Test
public void boolNullEveryOther() throws Exception {
final int count = 1024;
try (
BitVector in = new BitVector("in", allocator);
BitVector out = new BitVector("out", allocator);
) {
in.allocateNew(count);
ArrowBuf tempBuf = allocator.buffer(1024);
for (int i = 0; i < count; i ++) {
if (i % 2 == 0) {
in.set(i, 1);
}
}
in.setValueCount(count);
final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out));
try (
final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth());
final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount());
) {
fbv.ensureAvailableBlocks(count);
Pivots.pivot(pivot, count, fbv, vbv);
Unpivots.unpivot(pivot, fbv, vbv, 0, count);
for (int i = 0; i < count; i++) {
assertEquals(in.getObject(i), out.getObject(i));
}
}
tempBuf.release();
}
}
Example #9
| Source File: TestBoundedPivots.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private static Boolean[] populateBooleanValuesWithoutNull(BitVector vector, int size) {
assert size >= 4096;
vector.allocateNew(size);
Boolean[] booleanValues = new Boolean[size];
for (int i = 0; i < size; i++) {
if (i < 64 || (i >= 256 && i < 256 + 64)) {
vector.setNull(i);
continue;
}
if ((i % 6) != 0) {
/* every 6th value in boolean column is null */
if ((i & 1) == 0) {
/* column value true */
vector.set(i, 1);
booleanValues[i] = true;
} else {
/* column value false */
vector.set(i, 0);
booleanValues[i] = false;
}
} else {
vector.setNull(i);
booleanValues[i] = null;
}
}
vector.setValueCount(size);
return booleanValues;
}
Example #10
| Source File: TestBoundedPivots.java From dremio-oss with Apache License 2.0 | 5 votes |
|
@Test
public void fixedOnlyWithoutNull(){
try(IntVector col1 = new IntVector("col1", allocator);
BigIntVector col2 = new BigIntVector("col2", allocator);
DecimalVector col3 = new DecimalVector("col3", allocator, 30, 0);
BitVector col4 = new BitVector("col4", allocator)){
PivotDef pivot = PivotBuilder.getBlockDefinition(
new FieldVectorPair(col1, col1),
new FieldVectorPair(col2, col2),
new FieldVectorPair(col3, col3),
new FieldVectorPair(col4, col4)
);
Integer col1Val[] = populate4ByteValuesWithoutNull(col1, 4096);
Long col2Val[] = populate8ByteValuesWithoutNull(col2, 4096);
BigDecimal col3Val[] = populate16ByteValuesWithoutNull(col3, 4096);
Boolean col4Val[] = populateBooleanValuesWithoutNull(col4, 4096);
assertEquals(32, pivot.getBlockWidth());
try(FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth(), 4096, true);
VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount(), 4096 * 10, true)){
fixedOnlyHelper(pivot, fixed, variable, 0, 4096, false, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 128, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 17, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 76, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 39, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 189, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 123, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 1023, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 2046, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 3069, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 4092, 4, true, col1Val, col2Val, col3Val, col4Val);
}
}
}
Example #11
| Source File: TestBoundedPivots.java From dremio-oss with Apache License 2.0 | 5 votes |
|
@Test
public void fixedOnly(){
try(IntVector col1 = new IntVector("col1", allocator);
BigIntVector col2 = new BigIntVector("col2", allocator);
DecimalVector col3 = new DecimalVector("col3", allocator, 30, 0);
BitVector col4 = new BitVector("col4", allocator)){
PivotDef pivot = PivotBuilder.getBlockDefinition(
new FieldVectorPair(col1, col1),
new FieldVectorPair(col2, col2),
new FieldVectorPair(col3, col3),
new FieldVectorPair(col4, col4)
);
Integer col1Val[] = populate4ByteValues(col1, 4096);
Long col2Val[] = populate8ByteValues(col2, 4096);
BigDecimal col3Val[] = populate16ByteValues(col3, 4096);
Boolean col4Val[] = populateBooleanValues(col4, 4096);
assertEquals(32, pivot.getBlockWidth());
try(FixedBlockVector fixed = new FixedBlockVector(allocator, pivot.getBlockWidth(), 4096, true);
VariableBlockVector variable = new VariableBlockVector(allocator, pivot.getVariableCount(), 4096 * 10, true)){
fixedOnlyHelper(pivot, fixed, variable, 0, 4096, false, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 128, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 17, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 76, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 39, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 189, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 5, 123, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 0, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 1023, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 2046, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 3069, 1023, true, col1Val, col2Val, col3Val, col4Val);
fixedOnlyHelper(pivot, fixed, variable, 4092, 4, true, col1Val, col2Val, col3Val, col4Val);
}
}
}
Example #12
| Source File: TestPivotRoundtrip.java From dremio-oss with Apache License 2.0 | 5 votes |
|
@Test
public void boolNullEveryOther() throws Exception {
final int count = 1024;
try (
BitVector in = new BitVector("in", allocator);
BitVector out = new BitVector("out", allocator);
) {
in.allocateNew(count);
ArrowBuf tempBuf = allocator.buffer(1024);
for (int i = 0; i < count; i ++) {
if (i % 2 == 0) {
in.set(i, 1);
}
}
in.setValueCount(count);
final PivotDef pivot = PivotBuilder.getBlockDefinition(new FieldVectorPair(in, out));
try (
final FixedBlockVector fbv = new FixedBlockVector(allocator, pivot.getBlockWidth());
final VariableBlockVector vbv = new VariableBlockVector(allocator, pivot.getVariableCount());
) {
fbv.ensureAvailableBlocks(count);
Pivots.pivot(pivot, count, fbv, vbv);
Unpivots.unpivot(pivot, fbv, vbv, 0, count);
for (int i = 0; i < count; i++) {
assertEquals(in.getObject(i), out.getObject(i));
}
}
tempBuf.release();
}
}
Example #13
| Source File: TestBatchSize.java From dremio-oss with Apache License 2.0 | 5 votes |
|
@Test
public void testRecordBatchAllocs() {
OptionManager options = Mockito.mock(OptionManager.class);
when(options.getOption(ExecConstants.TARGET_BATCH_RECORDS_MIN)).thenReturn(127L);
when(options.getOption(ExecConstants.TARGET_BATCH_RECORDS_MAX)).thenReturn(4095L);
when(options.getOption(ExecConstants.TARGET_BATCH_SIZE_BYTES)).thenReturn(1024 * 1024L);
int batchSize = PhysicalPlanCreator.calculateBatchCountFromRecordSize(options, 0);
Assert.assertTrue(batchSize >= (int)(0.95 * 4096));
try (BufferAllocator allocator = allocatorRule.newAllocator("test-batch-size", 0, Long.MAX_VALUE)) {
final ValueVector bitV = new BitVector("bits", allocator);
AllocationHelper.allocate(bitV, batchSize, 0);
Assert.assertEquals(1024, allocator.getAllocatedMemory());
bitV.close();
final ValueVector intV = new IntVector("ints", allocator);
AllocationHelper.allocate(intV, batchSize, 0);
Assert.assertEquals(4096 * 4, allocator.getAllocatedMemory());
intV.close();
final ValueVector longV = new BigIntVector("longs", allocator);
AllocationHelper.allocate(longV, batchSize, 0);
Assert.assertEquals(4096 * 8, allocator.getAllocatedMemory());
longV.close();
final ValueVector decimalV = new DecimalVector("decimals", allocator, 38, 9);
AllocationHelper.allocate(decimalV, batchSize, 0);
Assert.assertEquals(4096 * 16, allocator.getAllocatedMemory());
decimalV.close();
}
}
Example #14
| Source File: TestQueryReAttempt.java From dremio-oss with Apache License 2.0 | 5 votes |
|
private BitVector bitVector(String name) {
BitVector vec = new BitVector(name, getAllocator());
vec.allocateNew(COUNT);
vec.set(0, 1);
vec.set(1, 0);
vec.setNull(2);
vec.set(3, 1);
vec.set(4, 1);
vec.setValueCount(COUNT);
return vec;
}
Example #15
| Source File: TestArrowBooleanConnector.java From multiple-dimension-spread with Apache License 2.0 | 5 votes |
|
@Test
public void T_convert_1() throws IOException{
BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
BitVector vector = new BitVector( "test" , allocator );
vector.allocateNew();
vector.setSafe( 0 , 0 );
vector.setSafe( 1 , 1 );
vector.setSafe( 2 , 0 );
vector.setNull( 3 );
vector.setSafe( 4 , 1 );
vector.setSafe( 5 , 1 );
vector.setSafe( 6 , 1 );
vector.setNull( 7 );
vector.setValueCount( 8 );
IColumn column = ArrowColumnFactory.convert( "test" , vector );
assertEquals( column.getColumnName() , "test" );
assertEquals( column.size() , 8 );
assertTrue( ( column.getColumnType() == ColumnType.BOOLEAN ) );
assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getBoolean() , false );
assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getBoolean() , false );
assertEquals( column.get(3).getRow() , null );
assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getBoolean() , true );
assertEquals( column.get(7).getRow() , null );
}
Example #16
| Source File: TestArrowBooleanConnector.java From yosegi with Apache License 2.0 | 5 votes |
|
@Test
public void T_convert_1() throws IOException{
BufferAllocator allocator = new RootAllocator( 1024 * 1024 * 10 );
BitVector vector = new BitVector( "test" , allocator );
vector.allocateNew();
vector.setSafe( 0 , 0 );
vector.setSafe( 1 , 1 );
vector.setSafe( 2 , 0 );
vector.setNull( 3 );
vector.setSafe( 4 , 1 );
vector.setSafe( 5 , 1 );
vector.setSafe( 6 , 1 );
vector.setNull( 7 );
vector.setValueCount( 8 );
IColumn column = ArrowColumnFactory.convert( "test" , vector );
assertEquals( column.getColumnName() , "test" );
assertEquals( column.size() , 8 );
assertTrue( ( column.getColumnType() == ColumnType.BOOLEAN ) );
assertEquals( ( (PrimitiveObject)( column.get(0).getRow() ) ).getBoolean() , false );
assertEquals( ( (PrimitiveObject)( column.get(1).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(2).getRow() ) ).getBoolean() , false );
assertEquals( column.get(3).getRow() , null );
assertEquals( ( (PrimitiveObject)( column.get(4).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(5).getRow() ) ).getBoolean() , true );
assertEquals( ( (PrimitiveObject)( column.get(6).getRow() ) ).getBoolean() , true );
assertEquals( column.get(7).getRow() , null );
}
Example #17
| Source File: ArrowVectorAccessors.java From iceberg with Apache License 2.0 | 5 votes |
|
@NotNull
@SuppressWarnings("checkstyle:CyclomaticComplexity")
private static ArrowVectorAccessor getPlainVectorAccessor(FieldVector vector) {
if (vector instanceof BitVector) {
return new BooleanAccessor((BitVector) vector);
} else if (vector instanceof IntVector) {
return new IntAccessor((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return new LongAccessor((BigIntVector) vector);
} else if (vector instanceof Float4Vector) {
return new FloatAccessor((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return new DoubleAccessor((Float8Vector) vector);
} else if (vector instanceof IcebergArrowVectors.DecimalArrowVector) {
return new DecimalAccessor((IcebergArrowVectors.DecimalArrowVector) vector);
} else if (vector instanceof IcebergArrowVectors.VarcharArrowVector) {
return new StringAccessor((IcebergArrowVectors.VarcharArrowVector) vector);
} else if (vector instanceof VarBinaryVector) {
return new BinaryAccessor((VarBinaryVector) vector);
} else if (vector instanceof DateDayVector) {
return new DateAccessor((DateDayVector) vector);
} else if (vector instanceof TimeStampMicroTZVector) {
return new TimestampAccessor((TimeStampMicroTZVector) vector);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
return new ArrayAccessor(listVector);
} else if (vector instanceof StructVector) {
StructVector structVector = (StructVector) vector;
return new StructAccessor(structVector);
}
throw new UnsupportedOperationException("Unsupported vector: " + vector.getClass());
}
Example #18
| Source File: ArrowUtils.java From flink with Apache License 2.0 | 4 votes |
|
private static ArrowFieldWriter<RowData> createArrowFieldWriterForRow(ValueVector vector, LogicalType fieldType) {
if (vector instanceof TinyIntVector) {
return TinyIntWriter.forRow((TinyIntVector) vector);
} else if (vector instanceof SmallIntVector) {
return SmallIntWriter.forRow((SmallIntVector) vector);
} else if (vector instanceof IntVector) {
return IntWriter.forRow((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return BigIntWriter.forRow((BigIntVector) vector);
} else if (vector instanceof BitVector) {
return BooleanWriter.forRow((BitVector) vector);
} else if (vector instanceof Float4Vector) {
return FloatWriter.forRow((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return DoubleWriter.forRow((Float8Vector) vector);
} else if (vector instanceof VarCharVector) {
return VarCharWriter.forRow((VarCharVector) vector);
} else if (vector instanceof VarBinaryVector) {
return VarBinaryWriter.forRow((VarBinaryVector) vector);
} else if (vector instanceof DecimalVector) {
DecimalVector decimalVector = (DecimalVector) vector;
return DecimalWriter.forRow(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
} else if (vector instanceof DateDayVector) {
return DateWriter.forRow((DateDayVector) vector);
} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
return TimeWriter.forRow(vector);
} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
int precision;
if (fieldType instanceof LocalZonedTimestampType) {
precision = ((LocalZonedTimestampType) fieldType).getPrecision();
} else {
precision = ((TimestampType) fieldType).getPrecision();
}
return TimestampWriter.forRow(vector, precision);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
LogicalType elementType = ((ArrayType) fieldType).getElementType();
return ArrayWriter.forRow(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
} else if (vector instanceof StructVector) {
RowType rowType = (RowType) fieldType;
ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
for (int i = 0; i < fieldsWriters.length; i++) {
fieldsWriters[i] = createArrowFieldWriterForRow(
((StructVector) vector).getVectorById(i),
rowType.getTypeAt(i));
}
return RowWriter.forRow((StructVector) vector, fieldsWriters);
} else {
throw new UnsupportedOperationException(String.format(
"Unsupported type %s.", fieldType));
}
}
Example #19
| Source File: HiveFieldConverter.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
final boolean value = (boolean) ((BooleanObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
((BitVector) outputVV).setSafe(outputIndex, value ? 1 : 0);
}
Example #20
| Source File: HiveORCCopiers.java From dremio-oss with Apache License 2.0 | 4 votes |
|
BitCopier(LongColumnVector inputVector, BitVector outputVector) {
this.inputVector = inputVector;
this.outputVector = outputVector;
}
Example #21
| Source File: HiveFieldConverter.java From dremio-oss with Apache License 2.0 | 4 votes |
|
@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
final boolean value = (boolean) ((BooleanObjectInspector)oi).getPrimitiveJavaObject(hiveFieldValue);
((BitVector) outputVV).setSafe(outputIndex, value ? 1 : 0);
}
Example #22
| Source File: BitToBooleanConverter.java From snowflake-jdbc with Apache License 2.0 | 4 votes |
|
public BitToBooleanConverter(ValueVector fieldVector, int columnIndex, DataConversionContext context)
{
super(SnowflakeType.BOOLEAN.name(), fieldVector, columnIndex, context);
this.bitVector = (BitVector) fieldVector;
}
Example #23
| Source File: BitToBooleanConverterTest.java From snowflake-jdbc with Apache License 2.0 | 4 votes |
|
@Test
public void testConvertToString() throws SFException
{
final int rowCount = 1000;
List<Boolean> expectedValues = new ArrayList<>();
Set<Integer> nullValIndex = new HashSet<>();
for (int i = 0; i < rowCount; i++)
{
expectedValues.add(random.nextBoolean());
}
Map<String, String> customFieldMeta = new HashMap<>();
customFieldMeta.put("logicalType", "BOOLEAN");
FieldType fieldType = new FieldType(true,
Types.MinorType.BIT.getType(),
null, customFieldMeta);
BitVector vector = new BitVector("col_one", fieldType,
allocator);
for (int i = 0; i < rowCount; i++)
{
boolean isNull = random.nextBoolean();
if (isNull)
{
vector.setNull(i);
nullValIndex.add(i);
}
else
{
vector.setSafe(i, expectedValues.get(i) ? 1 : 0);
}
}
ArrowVectorConverter converter = new BitToBooleanConverter(vector, 0, this);
for (int i = 0; i < rowCount; i++)
{
boolean boolVal = converter.toBoolean(i);
Object objectVal = converter.toObject(i);
String stringVal = converter.toString(i);
if (nullValIndex.contains(i))
{
assertThat(boolVal, is(false));
assertThat(objectVal, is(nullValue())); // current behavior
assertThat(stringVal, is(nullValue())); // current behavior
assertThat(converter.toBytes(i), is(nullValue()));
}
else
{
assertThat(boolVal, is(expectedValues.get(i)));
assertThat(objectVal, is(expectedValues.get(i)));
assertThat(stringVal, is(expectedValues.get(i).toString().toUpperCase()));
if (boolVal)
{
assertThat((byte) 0x1, is(converter.toBytes(i)[0]));
}
else
{
assertThat((byte) 0x0, is(converter.toBytes(i)[0]));
}
}
}
vector.clear();
}
Example #24
| Source File: ArrowBooleanColumnVector.java From flink with Apache License 2.0 | 4 votes |
|
public ArrowBooleanColumnVector(BitVector bitVector) {
this.bitVector = Preconditions.checkNotNull(bitVector);
}
Example #25
| Source File: BooleanFieldReader.java From flink with Apache License 2.0 | 4 votes |
|
public BooleanFieldReader(BitVector bitVector) {
super(bitVector);
}
Example #26
| Source File: BooleanFieldReader.java From flink with Apache License 2.0 | 4 votes |
|
@Override
public Boolean read(int index) {
return ((BitVector) getValueVector()).getObject(index);
}
Example #27
| Source File: ArrowUtils.java From flink with Apache License 2.0 | 4 votes |
|
private static ArrowFieldWriter<Row> createRowArrowFieldWriter(ValueVector vector, LogicalType fieldType) {
if (vector instanceof TinyIntVector) {
return new RowTinyIntWriter((TinyIntVector) vector);
} else if (vector instanceof SmallIntVector) {
return new RowSmallIntWriter((SmallIntVector) vector);
} else if (vector instanceof IntVector) {
return new RowIntWriter((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return new RowBigIntWriter((BigIntVector) vector);
} else if (vector instanceof BitVector) {
return new RowBooleanWriter((BitVector) vector);
} else if (vector instanceof Float4Vector) {
return new RowFloatWriter((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return new RowDoubleWriter((Float8Vector) vector);
} else if (vector instanceof VarCharVector) {
return new RowVarCharWriter((VarCharVector) vector);
} else if (vector instanceof VarBinaryVector) {
return new RowVarBinaryWriter((VarBinaryVector) vector);
} else if (vector instanceof DecimalVector) {
DecimalVector decimalVector = (DecimalVector) vector;
return new RowDecimalWriter(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
} else if (vector instanceof DateDayVector) {
return new RowDateWriter((DateDayVector) vector);
} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
return new RowTimeWriter(vector);
} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
return new RowTimestampWriter(vector);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
LogicalType elementType = ((ArrayType) fieldType).getElementType();
return new RowArrayWriter(listVector, createRowArrowFieldWriter(listVector.getDataVector(), elementType));
} else if (vector instanceof StructVector) {
RowType rowType = (RowType) fieldType;
ArrowFieldWriter<Row>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
for (int i = 0; i < fieldsWriters.length; i++) {
fieldsWriters[i] = createRowArrowFieldWriter(
((StructVector) vector).getVectorById(i),
rowType.getTypeAt(i));
}
return new RowRowWriter((StructVector) vector, fieldsWriters);
} else {
throw new UnsupportedOperationException(String.format(
"Unsupported type %s.", fieldType));
}
}
Example #28
| Source File: ArrowBooleanConnector.java From yosegi with Apache License 2.0 | 4 votes |
|
public ArrowBooleanConnector( final String columnName , final BitVector vector ) {
this.columnName = columnName;
this.vector = vector;
}
Example #29
| Source File: ArrowUtils.java From flink with Apache License 2.0 | 4 votes |
|
private static ArrowFieldWriter<ArrayData> createArrowFieldWriterForArray(ValueVector vector, LogicalType fieldType) {
if (vector instanceof TinyIntVector) {
return TinyIntWriter.forArray((TinyIntVector) vector);
} else if (vector instanceof SmallIntVector) {
return SmallIntWriter.forArray((SmallIntVector) vector);
} else if (vector instanceof IntVector) {
return IntWriter.forArray((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return BigIntWriter.forArray((BigIntVector) vector);
} else if (vector instanceof BitVector) {
return BooleanWriter.forArray((BitVector) vector);
} else if (vector instanceof Float4Vector) {
return FloatWriter.forArray((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return DoubleWriter.forArray((Float8Vector) vector);
} else if (vector instanceof VarCharVector) {
return VarCharWriter.forArray((VarCharVector) vector);
} else if (vector instanceof VarBinaryVector) {
return VarBinaryWriter.forArray((VarBinaryVector) vector);
} else if (vector instanceof DecimalVector) {
DecimalVector decimalVector = (DecimalVector) vector;
return DecimalWriter.forArray(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
} else if (vector instanceof DateDayVector) {
return DateWriter.forArray((DateDayVector) vector);
} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
return TimeWriter.forArray(vector);
} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
int precision;
if (fieldType instanceof LocalZonedTimestampType) {
precision = ((LocalZonedTimestampType) fieldType).getPrecision();
} else {
precision = ((TimestampType) fieldType).getPrecision();
}
return TimestampWriter.forArray(vector, precision);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
LogicalType elementType = ((ArrayType) fieldType).getElementType();
return ArrayWriter.forArray(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
} else if (vector instanceof StructVector) {
RowType rowType = (RowType) fieldType;
ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
for (int i = 0; i < fieldsWriters.length; i++) {
fieldsWriters[i] = createArrowFieldWriterForRow(
((StructVector) vector).getVectorById(i),
rowType.getTypeAt(i));
}
return RowWriter.forArray((StructVector) vector, fieldsWriters);
} else {
throw new UnsupportedOperationException(String.format(
"Unsupported type %s.", fieldType));
}
}
Example #30
| Source File: ArrowUtils.java From flink with Apache License 2.0 | 4 votes |
|
public static ArrowFieldReader createRowArrowFieldReader(ValueVector vector, LogicalType fieldType) {
if (vector instanceof TinyIntVector) {
return new TinyIntFieldReader((TinyIntVector) vector);
} else if (vector instanceof SmallIntVector) {
return new SmallIntFieldReader((SmallIntVector) vector);
} else if (vector instanceof IntVector) {
return new IntFieldReader((IntVector) vector);
} else if (vector instanceof BigIntVector) {
return new BigIntFieldReader((BigIntVector) vector);
} else if (vector instanceof BitVector) {
return new BooleanFieldReader((BitVector) vector);
} else if (vector instanceof Float4Vector) {
return new FloatFieldReader((Float4Vector) vector);
} else if (vector instanceof Float8Vector) {
return new DoubleFieldReader((Float8Vector) vector);
} else if (vector instanceof VarCharVector) {
return new VarCharFieldReader((VarCharVector) vector);
} else if (vector instanceof VarBinaryVector) {
return new VarBinaryFieldReader((VarBinaryVector) vector);
} else if (vector instanceof DecimalVector) {
return new DecimalFieldReader((DecimalVector) vector);
} else if (vector instanceof DateDayVector) {
return new DateFieldReader((DateDayVector) vector);
} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
return new TimeFieldReader(vector);
} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
return new TimestampFieldReader(vector);
} else if (vector instanceof ListVector) {
ListVector listVector = (ListVector) vector;
LogicalType elementType = ((ArrayType) fieldType).getElementType();
return new ArrayFieldReader(listVector,
createRowArrowFieldReader(listVector.getDataVector(), elementType),
elementType);
} else if (vector instanceof StructVector) {
StructVector structVector = (StructVector) vector;
ArrowFieldReader[] fieldReaders = new ArrowFieldReader[structVector.size()];
for (int i = 0; i < fieldReaders.length; i++) {
fieldReaders[i] = createRowArrowFieldReader(structVector.getVectorById(i), ((RowType) fieldType).getTypeAt(i));
}
return new RowFieldReader(structVector, fieldReaders);
} else {
throw new UnsupportedOperationException(String.format(
"Unsupported type %s.", fieldType));
}
}
