org.apache.arrow.vector.DateMilliVector Java Examples

private static Pair<DateMilliVector, ResultVerifier> testDateMilliVector(final int startIndexInCurrentOutput, final int startIndexInJob) {
  DateMilliVector colDateV = new DateMilliVector("colDate", allocator);
  colDateV.allocateNew(5);
  colDateV.set(0, 234);
  colDateV.set(1, -2342);
  colDateV.setNull(2);
  colDateV.set(3, 384928359245L);
  colDateV.set(4, 2342893433L);

  ResultVerifier verifier = new ResultVerifier() {
    @Override
    public void verify(DataPOJO output) {
      int index = startIndexInCurrentOutput;
      assertEquals("1970-01-01", output.extractValue("colDate", index++));
      assertEquals("1969-12-31", output.extractValue("colDate", index++));
      assertNull(output.extractValue("colDate", index++));
      assertEquals("1982-03-14", output.extractValue("colDate", index++));
      assertEquals("1970-01-28", output.extractValue("colDate", index++));
    }
  };

  return Pair.of(colDateV, verifier);
}
 

@Before
public void setUp() {
  valueVector = new DateMilliVector("t", new RootAllocator());
  valueVector.allocateNew(3);
  valueVector.set(0, NON_NULL_VALUE);
  valueVector.set(1, DST_VALUE);
  valueVector.setNull(2);

  accessor = new DateMilliAccessor(valueVector, UTC_CALENDAR.getTimeZone());
}
 

CorruptionDetectingNullableDateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize,
    ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
    boolean fixedLength, DateMilliVector v, SchemaElement schemaElement)
        throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  dateVector = v;
}
 

DateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
                boolean fixedLength, DateMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  vector = v;
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final DateWritableV2 writeable = ((DateObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  ((DateMilliVector) outputVV).setSafe(outputIndex, writeable.getDays() * MILLIS_PER_DAY);
}
 

DateMilliCopier(LongColumnVector inputVector, DateMilliVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

@Override
public void setSafeValue(ObjectInspector oi, Object hiveFieldValue, ValueVector outputVV, int outputIndex) {
  final DateWritable writeable = ((DateObjectInspector)oi).getPrimitiveWritableObject(hiveFieldValue);
  ((DateMilliVector) outputVV).setSafe(outputIndex, writeable.get().toLocalDate().toEpochDay() * MILLIS_PER_DAY);
}
 

DateMilliCopier(LongColumnVector inputVector, DateMilliVector outputVector) {
  this.inputVector = inputVector;
  this.outputVector = outputVector;
}
 

public static SqlAccessor getSqlAccessor(ValueVector vector, TimeZone defaultTz) {
  final MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(vector.getField().getType());
  switch(type){
  case UNION:
    return new UnionSqlAccessor((UnionVector) vector);
  case TINYINT:
    return new TinyIntAccessor((TinyIntVector) vector);
  case UINT1:
    return new UInt1Accessor((UInt1Vector) vector);
  case UINT2:
    return new UInt2Accessor((UInt2Vector) vector);
  case SMALLINT:
    return new SmallIntAccessor((SmallIntVector) vector);
  case INT:
    return new IntAccessor((IntVector) vector);
  case UINT4:
    return new UInt4Accessor((UInt4Vector) vector);
  case FLOAT4:
    return new Float4Accessor((Float4Vector) vector);
  case INTERVALYEAR:
    return new IntervalYearAccessor((IntervalYearVector) vector);
  case TIMEMILLI:
    return new TimeMilliAccessor((TimeMilliVector) vector, defaultTz);
  case BIGINT:
    return new BigIntAccessor((BigIntVector) vector);
  case UINT8:
    return new UInt8Accessor((UInt8Vector) vector);
  case FLOAT8:
    return new Float8Accessor((Float8Vector) vector);
  case DATEMILLI:
    return new DateMilliAccessor((DateMilliVector) vector, defaultTz);
  case TIMESTAMPMILLI:
    return new TimeStampMilliAccessor((TimeStampMilliVector) vector, defaultTz);
  case INTERVALDAY:
    return new IntervalDayAccessor((IntervalDayVector) vector);
  case DECIMAL:
    return new DecimalAccessor((DecimalVector) vector);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryAccessor((FixedSizeBinaryVector) vector);
  case VARBINARY:
    return new VarBinaryAccessor((VarBinaryVector) vector);
  case VARCHAR:
    return new VarCharAccessor((VarCharVector) vector);
  case BIT:
    return new BitAccessor((BitVector) vector);
  case STRUCT:
  case LIST:
    return new GenericAccessor(vector);
  }
  throw new UnsupportedOperationException(String.format("Unable to find sql accessor for minor type [%s]", type));
}
 

public DateMilliAccessor(DateMilliVector vector, TimeZone defaultTZ) {
  this.ac = vector;
  this.defaultTimeZone = Preconditions.checkNotNull(defaultTZ, "Null TimeZone supplied.");
}
 

@Override
public void set(ValueVector v, int index) {
  if(obj != null){
    ((DateMilliVector) v).setSafe(index,  obj.toDateTimeAtStartOfDay(DateTimeZone.UTC).getMillis());
  }
}
 

CorruptionDetectingDateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
                              boolean fixedLength, DateMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  vector = v;
}
 

CorruptDateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
                  boolean fixedLength, DateMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
  vector = v;
}
 

private FieldWriter makeFieldWriter(FieldVector vector)
{
    Field field = vector.getField();
    String fieldName = field.getName();
    Types.MinorType fieldType = Types.getMinorTypeForArrowType(field.getType());
    Extractor extractor = extractors.get(fieldName);
    ConstraintProjector constraint = constraints.get(fieldName);
    FieldWriterFactory factory = fieldWriterFactories.get(fieldName);

    if (factory != null) {
        return factory.create(vector, extractor, constraint);
    }

    if (extractor == null) {
        throw new IllegalStateException("Missing extractor for field[" + fieldName + "]");
    }

    switch (fieldType) {
        case INT:
            return new IntFieldWriter((IntExtractor) extractor, (IntVector) vector, constraint);
        case BIGINT:
            return new BigIntFieldWriter((BigIntExtractor) extractor, (BigIntVector) vector, constraint);
        case DATEMILLI:
            return new DateMilliFieldWriter((DateMilliExtractor) extractor, (DateMilliVector) vector, constraint);
        case DATEDAY:
            return new DateDayFieldWriter((DateDayExtractor) extractor, (DateDayVector) vector, constraint);
        case TINYINT:
            return new TinyIntFieldWriter((TinyIntExtractor) extractor, (TinyIntVector) vector, constraint);
        case SMALLINT:
            return new SmallIntFieldWriter((SmallIntExtractor) extractor, (SmallIntVector) vector, constraint);
        case FLOAT4:
            return new Float4FieldWriter((Float4Extractor) extractor, (Float4Vector) vector, constraint);
        case FLOAT8:
            return new Float8FieldWriter((Float8Extractor) extractor, (Float8Vector) vector, constraint);
        case DECIMAL:
            return new DecimalFieldWriter((DecimalExtractor) extractor, (DecimalVector) vector, constraint);
        case BIT:
            return new BitFieldWriter((BitExtractor) extractor, (BitVector) vector, constraint);
        case VARCHAR:
            return new VarCharFieldWriter((VarCharExtractor) extractor, (VarCharVector) vector, constraint);
        case VARBINARY:
            return new VarBinaryFieldWriter((VarBinaryExtractor) extractor, (VarBinaryVector) vector, constraint);
        default:
            throw new RuntimeException(fieldType + " is not supported");
    }
}
 

NullableCorruptDateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
    boolean fixedLength, DateMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

NullableDateReader(DeprecatedParquetVectorizedReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData,
                   boolean fixedLength, DateMilliVector v, SchemaElement schemaElement) throws ExecutionSetupException {
  super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement);
}
 

public void set(int position, CompleteType type, ValueVector vector, int offset) {
  if (vector.isNull(offset)) {
    set(position, null);
    return;
  }

  switch (type.toMinorType()) {
    case TINYINT:
    case UINT1:
      setInteger(position, Integer.valueOf((Byte)(vector.getObject(offset))));
      break;
    case SMALLINT:
    case UINT2:
      setInteger(position, Integer.valueOf((Short)(vector.getObject(offset))));
      break;
    case INT:
    case UINT4:
      setInteger(position, (Integer)vector.getObject(offset));
      break;
    case UINT8:
    case BIGINT:
      setLong(position, (Long)(vector.getObject(offset)));
      break;
    case FLOAT4:
      setFloat(position, ((Float)(vector.getObject(offset))));
      break;
    case FLOAT8:
      setDouble(position, ((Double)(vector.getObject(offset))));
      break;
    case BIT:
      setBoolean(position, ((Boolean)(vector.getObject(offset))));
      break;
    case VARBINARY:
      setBytes(position, ((byte[])(vector.getObject(offset))));
      break;
    case DECIMAL9:
    case DECIMAL18:
    case DECIMAL28SPARSE:
    case DECIMAL38SPARSE:
    case DECIMAL28DENSE:
    case DECIMAL38DENSE:
    case DECIMAL:
      setBigDecimal(position, ((BigDecimal)(vector.getObject(offset))));
      break;

    case DATE:
      if (vector instanceof DateMilliVector) {
        setInteger(position, Math.toIntExact(TimeUnit.MILLISECONDS.toDays(((DateMilliVector) vector).get(offset))));
      } else {
        //TODO: needs further tuning
        set(position, null);
      }
      break;
    case TIME:
    case TIMETZ:
    case TIMESTAMPTZ:
    case TIMESTAMP:
    case INTERVAL:
    case INTERVALYEAR:
    case INTERVALDAY:
      if (vector instanceof  TimeStampMilliVector) {
        setLong(position, (((TimeStampMilliVector) vector).get(offset)) * 1000);
      } else {
        //TODO: needs further tuning
        set(position, null);
      }
      break;

    case VARCHAR:
    case FIXEDCHAR:
    case FIXED16CHAR:
    case FIXEDSIZEBINARY:
    case VAR16CHAR:
      setString(position, vector.getObject(offset).toString());
      break;


    case NULL:
    case MONEY:
    case LATE:
    case STRUCT:
    case LIST:
    case GENERIC_OBJECT:
    case UNION:
      throw new IllegalArgumentException("Unsupported type in partition data: " + type.toString());

  }
}
 

public Class<? extends FieldVector> getValueVectorClass(){
  switch (Types.getMinorTypeForArrowType(type)) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
      return StructVector.class;
  case LIST:
      return ListVector.class;
  case NULL:
      return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(String.format("Unable to determine vector class for type %s.", type));
}
 

public static FieldVector getNewVector(Field field, BufferAllocator allocator, CallBack callBack) {
  if (field.getType() instanceof ObjectType) {
    return new ObjectVector(field.getName(), allocator);
  }

  MinorType type = org.apache.arrow.vector.types.Types.getMinorTypeForArrowType(field.getType());

  List<Field> children = field.getChildren();

  switch (type) {

  case UNION:
    UnionVector unionVector = new UnionVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      unionVector.initializeChildrenFromFields(children);
    }
    return unionVector;
  case LIST:
    ListVector listVector = new ListVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      listVector.initializeChildrenFromFields(children);
    }
    return listVector;
  case STRUCT:
    StructVector structVector = new StructVector(field.getName(), allocator, callBack);
    if (!children.isEmpty()) {
      structVector.initializeChildrenFromFields(children);
    }
    return structVector;

  case NULL:
    return new ZeroVector();
  case TINYINT:
    return new TinyIntVector(field, allocator);
  case UINT1:
    return new UInt1Vector(field, allocator);
  case UINT2:
    return new UInt2Vector(field, allocator);
  case SMALLINT:
    return new SmallIntVector(field, allocator);
  case INT:
    return new IntVector(field, allocator);
  case UINT4:
    return new UInt4Vector(field, allocator);
  case FLOAT4:
    return new Float4Vector(field, allocator);
  case INTERVALYEAR:
    return new IntervalYearVector(field, allocator);
  case TIMEMILLI:
    return new TimeMilliVector(field, allocator);
  case BIGINT:
    return new BigIntVector(field, allocator);
  case UINT8:
    return new UInt8Vector(field, allocator);
  case FLOAT8:
    return new Float8Vector(field, allocator);
  case DATEMILLI:
    return new DateMilliVector(field, allocator);
  case TIMESTAMPMILLI:
    return new TimeStampMilliVector(field, allocator);
  case INTERVALDAY:
    return new IntervalDayVector(field, allocator);
  case DECIMAL:
    return new DecimalVector(field, allocator);
  case FIXEDSIZEBINARY:
    return new FixedSizeBinaryVector(field.getName(), allocator, WIDTH_ESTIMATE);
  case VARBINARY:
    return new VarBinaryVector(field, allocator);
  case VARCHAR:
    return new VarCharVector(field, allocator);
  case BIT:
    return new BitVector(field, allocator);
  default:
    break;
  }
  // All ValueVector types have been handled.
  throw new UnsupportedOperationException(buildErrorMessage("get new vector", type));
}
 

public static Class<?> getValueVectorClass(MinorType type) {
  switch (type) {
  case UNION:
    return UnionVector.class;
  case STRUCT:
    return StructVector.class;
  case LIST:
    return ListVector.class;
  case NULL:
    return ZeroVector.class;
  case TINYINT:
    return TinyIntVector.class;
  case UINT1:
    return UInt1Vector.class;
  case UINT2:
    return UInt2Vector.class;
  case SMALLINT:
    return SmallIntVector.class;
  case INT:
    return IntVector.class;
  case UINT4:
    return UInt4Vector.class;
  case FLOAT4:
    return Float4Vector.class;
  case INTERVALYEAR:
    return IntervalYearVector.class;
  case TIMEMILLI:
    return TimeMilliVector.class;
  case BIGINT:
    return BigIntVector.class;
  case UINT8:
    return UInt8Vector.class;
  case FLOAT8:
    return Float8Vector.class;
  case DATEMILLI:
    return DateMilliVector.class;
  case TIMESTAMPMILLI:
    return TimeStampMilliVector.class;
  case INTERVALDAY:
    return IntervalDayVector.class;
  case DECIMAL:
    return DecimalVector.class;
  case FIXEDSIZEBINARY:
    return FixedSizeBinaryVector.class;
  case VARBINARY:
    return VarBinaryVector.class;
  case VARCHAR:
    return VarCharVector.class;
  case BIT:
    return BitVector.class;
  default:
    break;
  }
  throw new UnsupportedOperationException(buildErrorMessage("get value vector class", type));
}
 

DateMilliAccessor(DateMilliVector vector) {
  super(vector);
  this.accessor = vector;
}
 

public FlightArrowColumnVector(ValueVector vector) {
  super(FlightArrowUtils.fromArrowField(vector.getField()));

  if (vector instanceof BitVector) {
    accessor = new BooleanAccessor((BitVector) vector);
  } else if (vector instanceof TinyIntVector) {
    accessor = new ByteAccessor((TinyIntVector) vector);
  } else if (vector instanceof SmallIntVector) {
    accessor = new ShortAccessor((SmallIntVector) vector);
  } else if (vector instanceof IntVector) {
    accessor = new IntAccessor((IntVector) vector);
  } else if (vector instanceof BigIntVector) {
    accessor = new LongAccessor((BigIntVector) vector);
  } else if (vector instanceof Float4Vector) {
    accessor = new FloatAccessor((Float4Vector) vector);
  } else if (vector instanceof Float8Vector) {
    accessor = new DoubleAccessor((Float8Vector) vector);
  } else if (vector instanceof DecimalVector) {
    accessor = new DecimalAccessor((DecimalVector) vector);
  } else if (vector instanceof VarCharVector) {
    accessor = new StringAccessor((VarCharVector) vector);
  } else if (vector instanceof VarBinaryVector) {
    accessor = new BinaryAccessor((VarBinaryVector) vector);
  } else if (vector instanceof DateDayVector) {
    accessor = new DateAccessor((DateDayVector) vector);
  } else if (vector instanceof DateMilliVector) {
    accessor = new DateMilliAccessor((DateMilliVector) vector);
  } else if (vector instanceof TimeStampMicroTZVector) {
    accessor = new TimestampAccessor((TimeStampMicroTZVector) vector);
  } else if (vector instanceof TimeStampMilliVector) {
    accessor = new TimestampMilliAccessor((TimeStampMilliVector) vector);
  } else if (vector instanceof ListVector) {
    ListVector listVector = (ListVector) vector;
    accessor = new ArrayAccessor(listVector);
  } else if (vector instanceof StructVector) {
    StructVector structVector = (StructVector) vector;
    accessor = new StructAccessor(structVector);

    childColumns = new FlightArrowColumnVector[structVector.size()];
    for (int i = 0; i < childColumns.length; ++i) {
      childColumns[i] = new FlightArrowColumnVector(structVector.getVectorById(i));
    }
  } else {
    System.out.println(vector);
    throw new UnsupportedOperationException();
  }
}
 

/**
 * In some filtering situations it can be useful to 'unset' a row as an indication to a later processing stage
 * that the row is irrelevant. The mechanism by which we 'unset' a row is actually field type specific and as such
 * this method is not supported for all field types.
 *
 * @param row The row number to unset in the provided Block.
 * @param block The Block where we'd like to unset the specified row.
 */
public static void unsetRow(int row, Block block)
{
    for (FieldVector vector : block.getFieldVectors()) {
        switch (vector.getMinorType()) {
            case TIMESTAMPMILLITZ:
                ((TimeStampMilliTZVector) vector).setNull(row);
                break;
            case DATEDAY:
                ((DateDayVector) vector).setNull(row);
                break;
            case DATEMILLI:
                ((DateMilliVector) vector).setNull(row);
                break;
            case TINYINT:
                ((TinyIntVector) vector).setNull(row);
                break;
            case UINT1:
                ((UInt1Vector) vector).setNull(row);
                break;
            case SMALLINT:
                ((SmallIntVector) vector).setNull(row);
                break;
            case UINT2:
                ((UInt2Vector) vector).setNull(row);
                break;
            case UINT4:
                ((UInt4Vector) vector).setNull(row);
                break;
            case INT:
                ((IntVector) vector).setNull(row);
                break;
            case UINT8:
                ((UInt8Vector) vector).setNull(row);
                break;
            case BIGINT:
                ((BigIntVector) vector).setNull(row);
                break;
            case FLOAT4:
                ((Float4Vector) vector).setNull(row);
                break;
            case FLOAT8:
                ((Float8Vector) vector).setNull(row);
                break;
            case DECIMAL:
                ((DecimalVector) vector).setNull(row);
                break;
            case VARBINARY:
                ((VarBinaryVector) vector).setNull(row);
                break;
            case VARCHAR:
                ((VarCharVector) vector).setNull(row);
                break;
            case BIT:
                ((BitVector) vector).setNull(row);
                break;
            case STRUCT:
                ((StructVector) vector).setNull(row);
                break;
            case LIST:
                UnionListWriter writer = ((ListVector) vector).getWriter();
                writer.setPosition(row);
                writer.startList();
                writer.endList();
                writer.setValueCount(0);
                break;
            default:
                throw new IllegalArgumentException("Unknown type " + vector.getMinorType());
        }
    }
}
 

/**
 * Used to mark a particular cell as null.
 *
 * @param vector The FieldVector to write the null value to.
 * @param pos The position (row) in the FieldVector to mark as null.
 */
private static void setNullValue(FieldVector vector, int pos)
{
    switch (vector.getMinorType()) {
        case TIMESTAMPMILLITZ:
            ((TimeStampMilliTZVector) vector).setNull(pos);
            break;
        case DATEMILLI:
            ((DateMilliVector) vector).setNull(pos);
            break;
        case DATEDAY:
            ((DateDayVector) vector).setNull(pos);
            break;
        case FLOAT8:
            ((Float8Vector) vector).setNull(pos);
            break;
        case FLOAT4:
            ((Float4Vector) vector).setNull(pos);
            break;
        case INT:
            ((IntVector) vector).setNull(pos);
            break;
        case TINYINT:
            ((TinyIntVector) vector).setNull(pos);
            break;
        case SMALLINT:
            ((SmallIntVector) vector).setNull(pos);
            break;
        case UINT1:
            ((UInt1Vector) vector).setNull(pos);
            break;
        case UINT2:
            ((UInt2Vector) vector).setNull(pos);
            break;
        case UINT4:
            ((UInt4Vector) vector).setNull(pos);
            break;
        case UINT8:
            ((UInt8Vector) vector).setNull(pos);
            break;
        case BIGINT:
            ((BigIntVector) vector).setNull(pos);
            break;
        case VARBINARY:
            ((VarBinaryVector) vector).setNull(pos);
            break;
        case DECIMAL:
            ((DecimalVector) vector).setNull(pos);
            break;
        case VARCHAR:
            ((VarCharVector) vector).setNull(pos);
            break;
        case BIT:
            ((BitVector) vector).setNull(pos);
            break;
        default:
            throw new IllegalArgumentException("Unknown type " + vector.getMinorType());
    }
}