org.apache.arrow.vector.complex.UnionVector Java Examples

The following examples show how to use org.apache.arrow.vector.complex.UnionVector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TypeHelper.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
public static void loadFromValidityAndDataBuffers(ValueVector v, SerializedField metadata, ArrowBuf dataBuffer,
    ArrowBuf validityBuffer) {
  if (v instanceof ZeroVector) {
    throw new UnsupportedOperationException(String.format("this loader is not supported for vector %s", v));
  } else if (v instanceof UnionVector) {
    throw new UnsupportedOperationException(String.format("this loader is not supported for vector %s", v));
  } else if (v instanceof ListVector) {
    throw new UnsupportedOperationException(String.format("this loader is not supported for vector %s", v));
  } else if (v instanceof StructVector) {
    throw new UnsupportedOperationException(String.format("this loader is not supported for vector %s", v));
  } else if (v instanceof NonNullableStructVector) {
    throw new UnsupportedOperationException(String.format("this loader is not supported for vector %s", v));
  }

  Optional<ValueVectorHelper> helper = getHelper(v);

  if (!helper.isPresent()) {
    throw new UnsupportedOperationException(String.format("no loader for vector %s", v));
  }

  helper.get().loadFromValidityAndDataBuffers(metadata, dataBuffer, validityBuffer);
}
Example #2
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
UnionCopier(HiveColumnVectorData columnVectorData,
            int ordinalId,
            UnionColumnVector inputVector,
            UnionVector outputVector,
            HiveOperatorContextOptions operatorContextOptions) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
  // The loop below assumes that the getChildrenFromFields() API returns
  // the list of children in the same order as was provided when building the UnionVector.
  List<FieldVector> childArrowFields = outputVector.getChildrenFromFields();
  int childPos = ordinalId + 1; // first field is immediately next to union vector itself
  for (int idx=0; idx<childArrowFields.size(); ++idx) {
    if (idx < inputVector.fields.length) {
      ColumnVector hiveFieldVector = inputVector.fields[idx];
      ValueVector arrowfieldVector = childArrowFields.get(idx);
      arrowFieldVectors.add(arrowfieldVector);
      ORCCopier childCopier = createCopier(columnVectorData, childPos, arrowfieldVector, hiveFieldVector, operatorContextOptions);
      fieldCopiers.add(childCopier);
      childPos += columnVectorData.getTotalVectorCount(childPos);
    } else {
      fieldCopiers.add(new NoOpCopier(null, null));
    }
  }
}
Example #3
Source File: HiveORCCopiers.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
UnionCopier(HiveColumnVectorData columnVectorData,
            int ordinalId,
            UnionColumnVector inputVector,
            UnionVector outputVector,
            HiveOperatorContextOptions operatorContextOptions) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
  // The loop below assumes that the getChildrenFromFields() API returns
  // the list of children in the same order as was provided when building the UnionVector.
  List<FieldVector> childArrowFields = outputVector.getChildrenFromFields();
  int childPos = ordinalId + 1; // first field is immediately next to union vector itself
  for (int idx=0; idx<childArrowFields.size(); ++idx) {
    if (idx < inputVector.fields.length) {
      ColumnVector hiveFieldVector = inputVector.fields[idx];
      ValueVector arrowfieldVector = childArrowFields.get(idx);
      arrowFieldVectors.add(arrowfieldVector);
      ORCCopier childCopier = createCopier(columnVectorData, childPos, arrowfieldVector, hiveFieldVector, operatorContextOptions);
      fieldCopiers.add(childCopier);
      childPos += columnVectorData.getTotalVectorCount(childPos);
    } else {
      fieldCopiers.add(new NoOpCopier(null, null));
    }
  }
}
Example #4
Source File: TypeHelper.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
private static ValueVectorHelper getHelperNull(ValueVector v) {
  if (v instanceof ZeroVector) {
    return new ZeroVectorHelper((ZeroVector) v);
  } else if (v instanceof NullVector) {
    return new NullVectorHelper((NullVector) v);
  } else if (v instanceof UnionVector) {
    return new UnionVectorHelper((UnionVector) v);
  } else if (v instanceof ListVector) {
    return new ListVectorHelper((ListVector) v);
  } else if (v instanceof StructVector) {
    return new StructVectorHelper((StructVector) v);
  } else if (v instanceof NonNullableStructVector) {
    return new NonNullableStructVectorHelper((NonNullableStructVector) v);
  } else if (v instanceof BaseFixedWidthVector) {
    return new FixedWidthVectorHelper<BaseFixedWidthVector>((BaseFixedWidthVector) v);
  } else if (v instanceof BaseVariableWidthVector) {
    return new VariableWidthVectorHelper<BaseVariableWidthVector>((BaseVariableWidthVector) v);
  }

  return null;
}
Example #5
Source File: ArrowMemoryAllocatorFactory.java    From multiple-dimension-spread with Apache License 2.0 5 votes vote down vote up 
public static IMemoryAllocator getFromUnionVector( final ColumnType columnType , final String columnName , final BufferAllocator allocator , final UnionVector vector , final int rowCount ){
  switch( columnType ){
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      return new ArrowArrayMemoryAllocator( allocator , vector.getList() , rowCount );
    case SPREAD:
      return new ArrowMapMemoryAllocator( allocator , vector.getStruct() , rowCount );

    case BOOLEAN:
      return new ArrowBooleanMemoryAllocator( vector.getBitVector() , rowCount );
    case BYTE:
      return new ArrowByteMemoryAllocator( vector.getTinyIntVector() , rowCount );
    case SHORT:
      return new ArrowShortMemoryAllocator( vector.getSmallIntVector() , rowCount );
    case INTEGER:
      return new ArrowIntegerMemoryAllocator( vector.getIntVector() , rowCount );
    case LONG:
      return new ArrowLongMemoryAllocator( vector.getBigIntVector() , rowCount );
    case FLOAT:
      return new ArrowFloatMemoryAllocator( vector.getFloat4Vector() , rowCount );
    case DOUBLE:
      return new ArrowDoubleMemoryAllocator( vector.getFloat8Vector() , rowCount );
    case STRING:
      return new ArrowStringMemoryAllocator( vector.getVarCharVector() , rowCount );
    case BYTES:
      return new ArrowBytesMemoryAllocator( vector.getVarBinaryVector() , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example #6
Source File: ArrowUnionMemoryAllocator.java    From yosegi with Apache License 2.0 5 votes vote down vote up 
/**
 * Set the vector of Union and initialize it.
 */
public ArrowUnionMemoryAllocator(
    final BufferAllocator allocator , final UnionVector vector , final int rowCount ) {
  this.allocator = allocator;
  this.vector = vector;
  this.rowCount = rowCount;
  vector.allocateNew();
}
Example #7
Source File: TestArrowFileReader.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
/** Helper method which creates a union vector with no data */
private static UnionVector testEmptyUnionVector(BufferAllocator allocator) {
  final UnionVector unionVector = new UnionVector("unionVector", allocator, null);
  unionVector.initializeChildrenFromFields(
      asList(
          Field.nullable("intType", new ArrowType.Int(32, true)),
          Field.nullable("decimalType", new ArrowType.Decimal(4, 10))
      )
  );

  return unionVector;
}
Example #8
Source File: JobDataFragmentWrapper.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
@Override
public DataType extractType(String column, int index){
  Pair<RecordBatchData, Integer> dataBatch = find(index);
  Column columnDef = getColumn(column);
  ValueVector vv = dataBatch.getKey().getVectors().get(columnDef.getIndex());
  if (columnDef.getType() == DataType.MIXED) {
    final int typeValue = ((UnionVector)vv).getTypeValue(dataBatch.getValue());
    return DataTypeUtil.getDataType(getMinorTypeFromArrowMinorType(MinorType.values()[typeValue]));
  }

  return columnDef.getType();
}
Example #9
Source File: ArrowMemoryAllocatorFactory.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up 
public static IMemoryAllocator getFromStructVector( final ColumnType columnType , final String columnName , final BufferAllocator allocator , final StructVector vector , final int rowCount ){
  switch( columnType ){
    case UNION:
      UnionVector unionVector = vector.addOrGetUnion( columnName );
      return new ArrowUnionMemoryAllocator( allocator , unionVector , rowCount );
    case ARRAY:
      return new ArrowArrayMemoryAllocator( allocator , vector.addOrGetList( columnName ) , rowCount );
    case SPREAD:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowMapMemoryAllocator( allocator , mapVector , rowCount );

    case BOOLEAN:
      BitVector bitVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) , BitVector.class );
      return new ArrowBooleanMemoryAllocator( bitVector , rowCount );
    case BYTE:
      TinyIntVector byteVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) , TinyIntVector.class );
      return new ArrowByteMemoryAllocator( byteVector , rowCount );
    case SHORT:
      SmallIntVector shortVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 16 , true ) , null , null ) , SmallIntVector.class );
      return new ArrowShortMemoryAllocator( shortVector , rowCount );
    case INTEGER:
      IntVector integerVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) , IntVector.class );
      return new ArrowIntegerMemoryAllocator( integerVector , rowCount );
    case LONG:
      BigIntVector longVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.Int( 64 , true ) , null , null ) , BigIntVector.class );
      return new ArrowLongMemoryAllocator( longVector , rowCount );
    case FLOAT:
      Float4Vector floatVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.SINGLE ) , null , null ) , Float4Vector.class );
      return new ArrowFloatMemoryAllocator( floatVector , rowCount );
    case DOUBLE:
      Float8Vector doubleVector =  vector.addOrGet( columnName , new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) , null , null ) , Float8Vector.class );
      return new ArrowDoubleMemoryAllocator( doubleVector , rowCount );
    case STRING:
      VarCharVector charVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) , VarCharVector.class );
      return new ArrowStringMemoryAllocator( charVector , rowCount );
    case BYTES:
      VarBinaryVector binaryVector =  vector.addOrGet( columnName , new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) , VarBinaryVector.class );
      return new ArrowBytesMemoryAllocator( binaryVector , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example #10
Source File: ArrowMemoryAllocatorFactory.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up 
public static IMemoryAllocator getFromListVector( final ColumnType columnType , final String columnName , final BufferAllocator allocator , final ListVector vector , final int rowCount ){
  switch( columnType ){
    case UNION:
      AddOrGetResult<UnionVector> unionVector =  vector.addOrGetVector( new FieldType( true , MinorType.UNION.getType() , null , null ) );
      return new ArrowUnionMemoryAllocator( allocator , unionVector.getVector() , rowCount );
    case ARRAY:
      AddOrGetResult<ListVector> listVector =  vector.addOrGetVector( new FieldType( true , ArrowType.List.INSTANCE , null , null ) );
      return new ArrowArrayMemoryAllocator( allocator , listVector.getVector() , rowCount );
    case SPREAD:
      AddOrGetResult<StructVector> mapVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Struct.INSTANCE , null , null ) );
      return new ArrowMapMemoryAllocator( allocator , mapVector.getVector() , rowCount );

    case BOOLEAN:
      AddOrGetResult<BitVector> bitVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) );
      return new ArrowBooleanMemoryAllocator( bitVector.getVector() , rowCount );
    case BYTE:
      AddOrGetResult<TinyIntVector> byteVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) );
      return new ArrowByteMemoryAllocator( byteVector.getVector() , rowCount );
    case SHORT:
      AddOrGetResult<SmallIntVector> shortVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.Int( 16 , true ) , null , null ) );
      return new ArrowShortMemoryAllocator( shortVector.getVector() , rowCount );
    case INTEGER:
      AddOrGetResult<IntVector> integerVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) );
      return new ArrowIntegerMemoryAllocator( integerVector.getVector() , rowCount );
    case LONG:
      AddOrGetResult<BigIntVector> longVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.Int( 64 , true ) , null , null ) );
      return new ArrowLongMemoryAllocator( longVector.getVector() , rowCount );
    case FLOAT:
      AddOrGetResult<Float4Vector> floatVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.HALF ) , null , null ) );
      return new ArrowFloatMemoryAllocator( floatVector.getVector() , rowCount );
    case DOUBLE:
      AddOrGetResult<Float8Vector> doubleVector =  vector.addOrGetVector( new FieldType( true , new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) , null , null ) );
      return new ArrowDoubleMemoryAllocator( doubleVector.getVector() , rowCount );
    case STRING:
      AddOrGetResult<VarCharVector> charVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) );
      return new ArrowStringMemoryAllocator( charVector.getVector() , rowCount );
    case BYTES:
      AddOrGetResult<VarBinaryVector> binaryVector =  vector.addOrGetVector( new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) );
      return new ArrowBytesMemoryAllocator( binaryVector.getVector() , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example #11
Source File: ArrowUnionMemoryAllocator.java    From multiple-dimension-spread with Apache License 2.0 4 votes vote down vote up 
public ArrowUnionMemoryAllocator( final BufferAllocator allocator , final UnionVector vector , final int rowCount ){
  this.allocator = allocator;
  this.vector = vector;
  this.rowCount = rowCount;
  vector.allocateNew();
}
Example #12
Source File: BasicTypeHelper.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static Class<?> getValueVectorClass(MinorType type) {
  switch (type) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
    return StructVector.class;
  case LIST:
    return ListVector.class;
  case NULL:
    return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case FIXEDSIZEBINARY:
    return FixedSizeBinaryVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type));
}
Example #13
Source File: BasicTypeHelper.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static FieldVector getNewVector(Field field, BufferAllocator allocator, CallBack callBack) {
  if (field.getType() instanceof ObjectType) {
    return new ObjectVector(field.getName(), allocator);
  }

  MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(field.getType());

  List<Field> children = field.getChildren();

  switch (type) {

  case UNION:
    UnionVector unionVector = new UnionVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      unionVector.initializeChildrenFromFields(children);
    }
    return unionVector;
  case LIST:
    ListVector listVector = new ListVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      listVector.initializeChildrenFromFields(children);
    }
    return listVector;
  case STRUCT:
    StructVector structVector = new StructVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      structVector.initializeChildrenFromFields(children);
    }
    return structVector;

  case NULL:
    return new ZeroVector();
  case TINYINT:
    return new TinyIntVector(field, allocator);
  case UINT1:
    return new UInt1Vector(field, allocator);
  case UINT2:
    return new UInt2Vector(field, allocator);
  case SMALLINT:
    return new SmallIntVector(field, allocator);
  case INT:
    return new IntVector(field, allocator);
  case UINT4:
    return new UInt4Vector(field, allocator);
  case FLOAT4:
    return new Float4Vector(field, allocator);
  case INTERVALYEAR:
    return new IntervalYearVector(field, allocator);
  case TIMEMILLI:
    return new TimeMilliVector(field, allocator);
  case BIGINT:
    return new BigIntVector(field, allocator);
  case UINT8:
    return new UInt8Vector(field, allocator);
  case FLOAT8:
    return new Float8Vector(field, allocator);
  case DATEMILLI:
    return new DateMilliVector(field, allocator);
  case TIMESTAMPMILLI:
    return new TimeStampMilliVector(field, allocator);
  case INTERVALDAY:
    return new IntervalDayVector(field, allocator);
  case DECIMAL:
    return new DecimalVector(field, allocator);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryVector(field.getName(), allocator, WIDTH_ESTIMATE);
  case VARBINARY:
    return new VarBinaryVector(field, allocator);
  case VARCHAR:
    return new VarCharVector(field, allocator);
  case BIT:
    return new BitVector(field, allocator);
  default:
    break;
  }
  // All ValueVector types have been handled.
  throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
}
Example #14
Source File: ArrowMemoryAllocatorFactory.java    From yosegi with Apache License 2.0 4 votes vote down vote up 
/**
 * Set the vector of Struct and initialize it.
 */
public static IMemoryAllocator getFromStructVector(
    final ColumnType columnType ,
    final String columnName ,
    final BufferAllocator allocator ,
    final StructVector vector ,
    final int rowCount ) {
  switch ( columnType ) {
    case UNION:
      UnionVector unionVector = vector.addOrGetUnion( columnName );
      return new ArrowUnionMemoryAllocator( allocator , unionVector , rowCount );
    case ARRAY:
      return new ArrowArrayMemoryAllocator(
          allocator , vector.addOrGetList( columnName ) , rowCount );
    case SPREAD:
      StructVector mapVector = vector.addOrGetStruct( columnName );
      return new ArrowMapMemoryAllocator( allocator , mapVector , rowCount );

    case BOOLEAN:
      BitVector bitVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) ,
          BitVector.class );
      return new ArrowBooleanMemoryAllocator( bitVector , rowCount );
    case BYTE:
      TinyIntVector byteVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) ,
          TinyIntVector.class );
      return new ArrowByteMemoryAllocator( byteVector , rowCount );
    case SHORT:
      SmallIntVector shortVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.Int( 16 , true ) ,
            null ,
            null ) ,
          SmallIntVector.class );
      return new ArrowShortMemoryAllocator( shortVector , rowCount );
    case INTEGER:
      IntVector integerVector =  vector.addOrGet(
          columnName ,
          new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) ,
          IntVector.class );
      return new ArrowIntegerMemoryAllocator( integerVector , rowCount );
    case LONG:
      BigIntVector longVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.Int( 64 , true ) ,
            null ,
            null ) ,
            BigIntVector.class );
      return new ArrowLongMemoryAllocator( longVector , rowCount );
    case FLOAT:
      Float4Vector floatVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.SINGLE ) ,
            null ,
            null ) ,
          Float4Vector.class );
      return new ArrowFloatMemoryAllocator( floatVector , rowCount );
    case DOUBLE:
      Float8Vector doubleVector = vector.addOrGet(
          columnName ,
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) ,
            null ,
            null ) ,
          Float8Vector.class );
      return new ArrowDoubleMemoryAllocator( doubleVector , rowCount );
    case STRING:
      VarCharVector charVector = vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) ,
          VarCharVector.class );
      return new ArrowStringMemoryAllocator( charVector , rowCount );
    case BYTES:
      VarBinaryVector binaryVector = vector.addOrGet(
          columnName ,
          new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) ,
          VarBinaryVector.class );
      return new ArrowBytesMemoryAllocator( binaryVector , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example #15
Source File: CompleteType.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public Class<? extends FieldVector> getValueVectorClass(){
  switch (Types.getMinorTypeForArrowType(type)) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
      return StructVector.class;
  case LIST:
      return ListVector.class;
  case NULL:
      return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(String.format("Unable to determine vector class for type %s.", type));
}
Example #16
Source File: UnionSqlAccessor.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public UnionSqlAccessor(UnionVector vector) {
  reader = vector.getReader();
}
Example #17
Source File: SqlAccessorBuilder.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
public static SqlAccessor getSqlAccessor(ValueVector vector, TimeZone defaultTz) {
  final MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(vector.getField().getType());
  switch(type){
  case UNION:
    return new UnionSqlAccessor((UnionVector) vector);
  case TINYINT:
    return new TinyIntAccessor((TinyIntVector) vector);
  case UINT1:
    return new UInt1Accessor((UInt1Vector) vector);
  case UINT2:
    return new UInt2Accessor((UInt2Vector) vector);
  case SMALLINT:
    return new SmallIntAccessor((SmallIntVector) vector);
  case INT:
    return new IntAccessor((IntVector) vector);
  case UINT4:
    return new UInt4Accessor((UInt4Vector) vector);
  case FLOAT4:
    return new Float4Accessor((Float4Vector) vector);
  case INTERVALYEAR:
    return new IntervalYearAccessor((IntervalYearVector) vector);
  case TIMEMILLI:
    return new TimeMilliAccessor((TimeMilliVector) vector, defaultTz);
  case BIGINT:
    return new BigIntAccessor((BigIntVector) vector);
  case UINT8:
    return new UInt8Accessor((UInt8Vector) vector);
  case FLOAT8:
    return new Float8Accessor((Float8Vector) vector);
  case DATEMILLI:
    return new DateMilliAccessor((DateMilliVector) vector, defaultTz);
  case TIMESTAMPMILLI:
    return new TimeStampMilliAccessor((TimeStampMilliVector) vector, defaultTz);
  case INTERVALDAY:
    return new IntervalDayAccessor((IntervalDayVector) vector);
  case DECIMAL:
    return new DecimalAccessor((DecimalVector) vector);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryAccessor((FixedSizeBinaryVector) vector);
  case VARBINARY:
    return new VarBinaryAccessor((VarBinaryVector) vector);
  case VARCHAR:
    return new VarCharAccessor((VarCharVector) vector);
  case BIT:
    return new BitAccessor((BitVector) vector);
  case STRUCT:
  case LIST:
    return new GenericAccessor(vector);
  }
  throw new UnsupportedOperationException(String.format("Unable to find sql accessor for minor type [%s]", type));
}
Example #18
Source File: TestData.java    From dremio-oss with Apache License 2.0 4 votes vote down vote up 
private Pair<UnionVector, ResultVerifier> testUnionVector(final int startIndexInCurrentOutput, final int startIndexInJob) {
  UnionVector colUnionV = new UnionVector("colUnion", allocator, null);

  UnionWriter unionWriter = new UnionWriter(colUnionV);
  unionWriter.allocate();
  for (int i = 0; i < 4; i++) {
    unionWriter.setPosition(i);
    if (i % 2 == 0) {
      unionWriter.writeInt(i);
    } else {
      unionWriter.writeFloat4(i);
    }
  }
  unionWriter.setPosition(4);
  byte[] varCharVal = "union varchar union varchar union varchar".getBytes();
  tempBuf.setBytes(0, varCharVal);
  unionWriter.writeVarChar(0, varCharVal.length, tempBuf);

  colUnionV.setValueCount(5);

  ResultVerifier verifier = new ResultVerifier() {
    @Override
    public void verify(DataPOJO output) {
      int index = startIndexInCurrentOutput;
      int uIndex = startIndexInJob;
      assertEquals(DataType.INTEGER, output.extractType("colUnion", index));
      assertEquals(getExpectedNumber(0), output.extractValue("colUnion", index));
      assertNull(output.extractUrl("colUnion", index++));
      uIndex++;

      assertEquals(DataType.FLOAT, output.extractType("colUnion", index));
      verifyDoubleValue(1d, output, "colUnion", index, 0.01f);
      assertNull(output.extractUrl("colUnion", index++));
      uIndex++;

      assertEquals(DataType.INTEGER, output.extractType("colUnion", index));
      assertEquals(getExpectedNumber(index), output.extractValue("colUnion", index));
      assertNull(output.extractUrl("colUnion", index++));
      uIndex++;

      assertEquals(DataType.FLOAT, output.extractType("colUnion", index));
      verifyDoubleValue(3d, output, "colUnion", index, 0.01f);
      assertNull(output.extractUrl("colUnion", index++));
      uIndex++;

      assertEquals(DataType.TEXT, output.extractType("colUnion", index));
      assertEquals("union varchar union varchar un", output.extractValue("colUnion", index));
      assertEquals(cellUrl(uIndex++, "colUnion"), output.extractUrl("colUnion", index++));
    }
  };

  return Pair.of(colUnionV, verifier);
}
Example #19
Source File: ArrowMemoryAllocatorFactory.java    From yosegi with Apache License 2.0 4 votes vote down vote up 
/**
 * Set the vector of Union and initialize it.
 */
public static IMemoryAllocator getFromUnionVector(
    final ColumnType columnType ,
    final String columnName ,
    final BufferAllocator allocator ,
    final UnionVector vector ,
    final int rowCount ) {
  switch ( columnType ) {
    case UNION:
      return NullMemoryAllocator.INSTANCE;
    case ARRAY:
      return new ArrowArrayMemoryAllocator( allocator , vector.getList() , rowCount );
    case SPREAD:
      return new ArrowMapMemoryAllocator( allocator , vector.getStruct() , rowCount );

    case BOOLEAN:
      return new ArrowBooleanMemoryAllocator( vector.getBitVector() , rowCount );
    case BYTE:
      return new ArrowByteMemoryAllocator( vector.getTinyIntVector() , rowCount );
    case SHORT:
      return new ArrowShortMemoryAllocator( vector.getSmallIntVector() , rowCount );
    case INTEGER:
      return new ArrowIntegerMemoryAllocator( vector.getIntVector() , rowCount );
    case LONG:
      return new ArrowLongMemoryAllocator( vector.getBigIntVector() , rowCount );
    case FLOAT:
      return new ArrowFloatMemoryAllocator( vector.getFloat4Vector() , rowCount );
    case DOUBLE:
      return new ArrowDoubleMemoryAllocator( vector.getFloat8Vector() , rowCount );
    case STRING:
      return new ArrowStringMemoryAllocator( vector.getVarCharVector() , rowCount );
    case BYTES:
      return new ArrowBytesMemoryAllocator( vector.getVarBinaryVector() , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}
Example #20
Source File: ArrowMemoryAllocatorFactory.java    From yosegi with Apache License 2.0 4 votes vote down vote up 
/**
 * Set the vector of List and initialize it.
 */
public static IMemoryAllocator getFromListVector(
      final ColumnType columnType ,
      final String columnName ,
      final BufferAllocator allocator ,
      final ListVector vector ,
      final int rowCount ) {
  switch ( columnType ) {
    case UNION:
      AddOrGetResult<UnionVector> unionVector =  vector.addOrGetVector(
          new FieldType( true , MinorType.UNION.getType() , null , null ) );
      return new ArrowUnionMemoryAllocator( allocator , unionVector.getVector() , rowCount );
    case ARRAY:
      AddOrGetResult<ListVector> listVector =  vector.addOrGetVector(
          new FieldType( true , ArrowType.List.INSTANCE , null , null ) );
      return new ArrowArrayMemoryAllocator( allocator , listVector.getVector() , rowCount );
    case SPREAD:
      AddOrGetResult<StructVector> mapVector = vector.addOrGetVector(
          new FieldType( true , ArrowType.Struct.INSTANCE , null , null ) );
      return new ArrowMapMemoryAllocator( allocator , mapVector.getVector() , rowCount );

    case BOOLEAN:
      AddOrGetResult<BitVector> bitVector = vector.addOrGetVector(
          new FieldType( true , ArrowType.Bool.INSTANCE , null , null ) );
      return new ArrowBooleanMemoryAllocator( bitVector.getVector() , rowCount );
    case BYTE:
      AddOrGetResult<TinyIntVector> byteVector = vector.addOrGetVector(
          new FieldType( true , new ArrowType.Int( 8 , true ) , null , null ) );
      return new ArrowByteMemoryAllocator( byteVector.getVector() , rowCount );
    case SHORT:
      AddOrGetResult<SmallIntVector> shortVector = vector.addOrGetVector(
          new FieldType( true , new ArrowType.Int( 16 , true ) , null , null ) );
      return new ArrowShortMemoryAllocator( shortVector.getVector() , rowCount );
    case INTEGER:
      AddOrGetResult<IntVector> integerVector =  vector.addOrGetVector(
          new FieldType( true , new ArrowType.Int( 32 , true ) , null , null ) );
      return new ArrowIntegerMemoryAllocator( integerVector.getVector() , rowCount );
    case LONG:
      AddOrGetResult<BigIntVector> longVector =  vector.addOrGetVector(
          new FieldType( true , new ArrowType.Int( 64 , true ) , null , null ) );
      return new ArrowLongMemoryAllocator( longVector.getVector() , rowCount );
    case FLOAT:
      AddOrGetResult<Float4Vector> floatVector = vector.addOrGetVector(
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.HALF ) ,
            null ,
            null ) );
      return new ArrowFloatMemoryAllocator( floatVector.getVector() , rowCount );
    case DOUBLE:
      AddOrGetResult<Float8Vector> doubleVector = vector.addOrGetVector(
          new FieldType(
            true ,
            new ArrowType.FloatingPoint( FloatingPointPrecision.DOUBLE ) ,
            null ,
            null ) );
      return new ArrowDoubleMemoryAllocator( doubleVector.getVector() , rowCount );
    case STRING:
      AddOrGetResult<VarCharVector> charVector = vector.addOrGetVector(
          new FieldType( true , ArrowType.Utf8.INSTANCE , null , null ) );
      return new ArrowStringMemoryAllocator( charVector.getVector() , rowCount );
    case BYTES:
      AddOrGetResult<VarBinaryVector> binaryVector = vector.addOrGetVector(
          new FieldType( true , ArrowType.Binary.INSTANCE , null , null ) );
      return new ArrowBytesMemoryAllocator( binaryVector.getVector() , rowCount );

    case NULL:
    case EMPTY_ARRAY:
    case EMPTY_SPREAD:
    default:
      return NullMemoryAllocator.INSTANCE;
  }
}