org.apache.flink.table.types.logical.LocalZonedTimestampType Java Examples

The following examples show how to use org.apache.flink.table.types.logical.LocalZonedTimestampType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LogicalTypeParser.java    From flink with Apache License 2.0 6 votes vote down vote up
private LogicalType parseTimestampType() {
	int precision = parseOptionalPrecision(TimestampType.DEFAULT_PRECISION);
	if (hasNextToken(Keyword.WITHOUT)) {
		nextToken(Keyword.WITHOUT);
		nextToken(Keyword.TIME);
		nextToken(Keyword.ZONE);
	} else if (hasNextToken(Keyword.WITH)) {
		nextToken(Keyword.WITH);
		if (hasNextToken(Keyword.LOCAL)) {
			nextToken(Keyword.LOCAL);
			nextToken(Keyword.TIME);
			nextToken(Keyword.ZONE);
			return new LocalZonedTimestampType(precision);
		} else {
			nextToken(Keyword.TIME);
			nextToken(Keyword.ZONE);
			return new ZonedTimestampType(precision);
		}
	}
	return new TimestampType(precision);
}
 
Example #2
Source File: DataFormatConverters.java    From flink with Apache License 2.0 6 votes vote down vote up
private static int getDateTimePrecision(LogicalType logicalType) {
	if (logicalType instanceof LocalZonedTimestampType) {
		return ((LocalZonedTimestampType) logicalType).getPrecision();
	} else if (logicalType instanceof TimestampType) {
		return ((TimestampType) logicalType).getPrecision();
	} else {
		TypeInformation typeInfo = ((LegacyTypeInformationType) logicalType).getTypeInformation();
		if (typeInfo instanceof LegacyInstantTypeInfo) {
			return ((LegacyInstantTypeInfo) typeInfo).getPrecision();
		} else if (typeInfo instanceof LegacyLocalDateTimeTypeInfo) {
			return ((LegacyLocalDateTimeTypeInfo) typeInfo).getPrecision();
		} else {
			// TimestampType.DEFAULT_PRECISION == LocalZonedTimestampType.DEFAULT_PRECISION == 6
			return TimestampType.DEFAULT_PRECISION;
		}
	}
}
 
Example #3
Source File: LogicalTypeParser.java    From flink with Apache License 2.0 6 votes vote down vote up
private LogicalType parseTimestampType() {
	int precision = parseOptionalPrecision(TimestampType.DEFAULT_PRECISION);
	if (hasNextToken(Keyword.WITHOUT)) {
		nextToken(Keyword.WITHOUT);
		nextToken(Keyword.TIME);
		nextToken(Keyword.ZONE);
	} else if (hasNextToken(Keyword.WITH)) {
		nextToken(Keyword.WITH);
		if (hasNextToken(Keyword.LOCAL)) {
			nextToken(Keyword.LOCAL);
			nextToken(Keyword.TIME);
			nextToken(Keyword.ZONE);
			return new LocalZonedTimestampType(precision);
		} else {
			nextToken(Keyword.TIME);
			nextToken(Keyword.ZONE);
			return new ZonedTimestampType(precision);
		}
	}
	return new TimestampType(precision);
}
 
Example #4
Source File: BinaryRowData.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * If it is a fixed-length field, we can call this BinaryRowData's setXX method for in-place updates.
 * If it is variable-length field, can't use this method, because the underlying data is stored continuously.
 */
public static boolean isInFixedLengthPart(LogicalType type) {
	switch (type.getTypeRoot()) {
		case BOOLEAN:
		case TINYINT:
		case SMALLINT:
		case INTEGER:
		case DATE:
		case TIME_WITHOUT_TIME_ZONE:
		case INTERVAL_YEAR_MONTH:
		case BIGINT:
		case INTERVAL_DAY_TIME:
		case FLOAT:
		case DOUBLE:
			return true;
		case DECIMAL:
			return DecimalData.isCompact(((DecimalType) type).getPrecision());
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			return TimestampData.isCompact(((TimestampType) type).getPrecision());
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			return TimestampData.isCompact(((LocalZonedTimestampType) type).getPrecision());
		default:
			return false;
	}
}
 
Example #5
Source File: LogicalTypeGeneralization.java    From flink with Apache License 2.0 5 votes vote down vote up
private static LogicalType createTimestampType(LogicalTypeRoot typeRoot, int precision) {
	switch (typeRoot) {
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			return new TimestampType(precision);
		case TIMESTAMP_WITH_TIME_ZONE:
			return new ZonedTimestampType(precision);
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			return new LocalZonedTimestampType(precision);
		default:
			throw new IllegalArgumentException();
	}
}
 
Example #6
Source File: LogicalTypesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalZonedTimestampTypeWithTimeAttribute() {
	testAll(
		new LocalZonedTimestampType(true, TimestampKind.ROWTIME, 9),
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		"TIMESTAMP(9) WITH LOCAL TIME ZONE *ROWTIME*",
		new Class[]{java.time.Instant.class, long.class, int.class},
		new Class[]{java.time.Instant.class},
		new LogicalType[]{},
		new LocalZonedTimestampType(3)
	);
}
 
Example #7
Source File: LogicalTypesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalZonedTimestampType() {
	testAll(
		new LocalZonedTimestampType(9),
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		new Class[]{java.time.Instant.class, long.class, int.class},
		new Class[]{java.time.Instant.class},
		new LogicalType[]{},
		new LocalZonedTimestampType(3)
	);
}
 
Example #8
Source File: FlinkTypeToType.java    From iceberg with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("checkstyle:CyclomaticComplexity")
@Override
public Type atomic(AtomicDataType type) {
  LogicalType inner = type.getLogicalType();
  if (inner instanceof VarCharType ||
      inner instanceof CharType) {
    return Types.StringType.get();
  } else if (inner instanceof BooleanType) {
    return Types.BooleanType.get();
  } else if (inner instanceof IntType ||
      inner instanceof SmallIntType ||
      inner instanceof TinyIntType) {
    return Types.IntegerType.get();
  } else if (inner instanceof BigIntType) {
    return Types.LongType.get();
  } else if (inner instanceof VarBinaryType) {
    return Types.BinaryType.get();
  } else if (inner instanceof BinaryType) {
    BinaryType binaryType = (BinaryType) inner;
    return Types.FixedType.ofLength(binaryType.getLength());
  } else if (inner instanceof FloatType) {
    return Types.FloatType.get();
  } else if (inner instanceof DoubleType) {
    return Types.DoubleType.get();
  } else if (inner instanceof DateType) {
    return Types.DateType.get();
  } else if (inner instanceof TimeType) {
    return Types.TimeType.get();
  } else if (inner instanceof TimestampType) {
    return Types.TimestampType.withoutZone();
  } else if (inner instanceof LocalZonedTimestampType) {
    return Types.TimestampType.withZone();
  } else if (inner instanceof DecimalType) {
    DecimalType decimalType = (DecimalType) inner;
    return Types.DecimalType.of(decimalType.getPrecision(), decimalType.getScale());
  } else {
    throw new UnsupportedOperationException("Not a supported type: " + type.toString());
  }
}
 
Example #9
Source File: LogicalTypesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalZonedTimestampTypeWithTimeAttribute() {
	testAll(
		new LocalZonedTimestampType(true, TimestampKind.ROWTIME, 9),
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		"TIMESTAMP(9) WITH LOCAL TIME ZONE *ROWTIME*",
		new Class[]{java.time.Instant.class, long.class, int.class},
		new Class[]{java.time.Instant.class},
		new LogicalType[]{},
		new LocalZonedTimestampType(3)
	);
}
 
Example #10
Source File: LogicalTypesTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalZonedTimestampType() {
	testAll(
		new LocalZonedTimestampType(9),
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		"TIMESTAMP(9) WITH LOCAL TIME ZONE",
		new Class[]{java.time.Instant.class, long.class, int.class},
		new Class[]{java.time.Instant.class},
		new LogicalType[]{},
		new LocalZonedTimestampType(3)
	);
}
 
Example #11
Source File: PythonTypeUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public FlinkFnApi.Schema.FieldType visit(LocalZonedTimestampType localZonedTimestampType) {
	FlinkFnApi.Schema.FieldType.Builder builder =
		FlinkFnApi.Schema.FieldType.newBuilder()
			.setTypeName(FlinkFnApi.Schema.TypeName.LOCAL_ZONED_TIMESTAMP)
			.setNullable(localZonedTimestampType.isNullable());

	FlinkFnApi.Schema.LocalZonedTimestampInfo.Builder dateTimeBuilder =
		FlinkFnApi.Schema.LocalZonedTimestampInfo.newBuilder()
			.setPrecision(localZonedTimestampType.getPrecision());
	builder.setLocalZonedTimestampInfo(dateTimeBuilder.build());
	return builder.build();
}
 
Example #12
Source File: LogicalTypeMerging.java    From flink with Apache License 2.0 5 votes vote down vote up
private static LogicalType createTimestampType(LogicalTypeRoot typeRoot, int precision) {
	switch (typeRoot) {
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			return new TimestampType(precision);
		case TIMESTAMP_WITH_TIME_ZONE:
			return new ZonedTimestampType(precision);
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			return new LocalZonedTimestampType(precision);
		default:
			throw new IllegalArgumentException();
	}
}
 
Example #13
Source File: ArrowUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ArrowType visit(LocalZonedTimestampType localZonedTimestampType) {
	if (localZonedTimestampType.getPrecision() == 0) {
		return new ArrowType.Timestamp(TimeUnit.SECOND, null);
	} else if (localZonedTimestampType.getPrecision() >= 1 && localZonedTimestampType.getPrecision() <= 3) {
		return new ArrowType.Timestamp(TimeUnit.MILLISECOND, null);
	} else if (localZonedTimestampType.getPrecision() >= 4 && localZonedTimestampType.getPrecision() <= 6) {
		return new ArrowType.Timestamp(TimeUnit.MICROSECOND, null);
	} else {
		return new ArrowType.Timestamp(TimeUnit.NANOSECOND, null);
	}
}
 
Example #14
Source File: PythonTypeUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public TypeSerializer visit(LocalZonedTimestampType localZonedTimestampType) {
	return new TimestampDataSerializer(localZonedTimestampType.getPrecision());
}
 
Example #15
Source File: LogicalTypeDefaultVisitor.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public R visit(LocalZonedTimestampType localZonedTimestampType) {
	return defaultMethod(localZonedTimestampType);
}
 
Example #16
Source File: LogicalTypeUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public LogicalType visit(LocalZonedTimestampType localZonedTimestampType) {
	return new LocalZonedTimestampType(
		localZonedTimestampType.isNullable(),
		localZonedTimestampType.getPrecision());
}
 
Example #17
Source File: LogicalTypeChecks.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public TimestampKind visit(LocalZonedTimestampType localZonedTimestampType) {
	return localZonedTimestampType.getKind();
}
 
Example #18
Source File: CsvRowDataSerializationSchema.java    From flink with Apache License 2.0 4 votes vote down vote up
private RowFieldConverter createRowFieldConverter(LogicalType fieldType) {
	switch (fieldType.getTypeRoot()) {
		case NULL:
			return (csvMapper, container, row, pos) -> container.nullNode();
		case BOOLEAN:
			return (csvMapper, container, row, pos) -> container.booleanNode(row.getBoolean(pos));
		case TINYINT:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getByte(pos));
		case SMALLINT:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getShort(pos));
		case INTEGER:
		case INTERVAL_YEAR_MONTH:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getInt(pos));
		case BIGINT:
		case INTERVAL_DAY_TIME:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getLong(pos));
		case FLOAT:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getFloat(pos));
		case DOUBLE:
			return (csvMapper, container, row, pos) -> container.numberNode(row.getDouble(pos));
		case CHAR:
		case VARCHAR:
			return (csvMapper, container, row, pos) -> container.textNode(row.getString(pos).toString());
		case BINARY:
		case VARBINARY:
			return (csvMapper, container, row, pos) -> container.binaryNode(row.getBinary(pos));
		case DATE:
			return (csvMapper, container, row, pos) -> convertDate(row.getInt(pos), container);
		case TIME_WITHOUT_TIME_ZONE:
			return (csvMapper, container, row, pos) -> convertTime(row.getInt(pos), container);
		case TIMESTAMP_WITH_TIME_ZONE:
			final int zonedTimestampPrecision = ((LocalZonedTimestampType) fieldType).getPrecision();
			return (csvMapper, container, row, pos) ->
				convertTimestamp(row.getTimestamp(pos, zonedTimestampPrecision), container);
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			final int timestampPrecision = ((TimestampType) fieldType).getPrecision();
			return (csvMapper, container, row, pos) ->
				convertTimestamp(row.getTimestamp(pos, timestampPrecision), container);
		case DECIMAL:
			return createDecimalRowFieldConverter((DecimalType) fieldType);
		case ARRAY:
			return createArrayRowFieldConverter((ArrayType) fieldType);
		case ROW:
			return createRowRowFieldConverter((RowType) fieldType);
		case MAP:
		case MULTISET:
		case RAW:
		default:
			throw new UnsupportedOperationException("Unsupported type: " + fieldType);
	}
}
 
Example #19
Source File: CsvRowDataSerializationSchema.java    From flink with Apache License 2.0 4 votes vote down vote up
private ArrayElementConverter createArrayElementConverter(LogicalType fieldType) {
	switch (fieldType.getTypeRoot()) {
		case NULL:
			return (csvMapper, container, array, pos) -> container.nullNode();
		case BOOLEAN:
			return (csvMapper, container, array, pos) -> container.booleanNode(array.getBoolean(pos));
		case TINYINT:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getByte(pos));
		case SMALLINT:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getShort(pos));
		case INTEGER:
		case INTERVAL_YEAR_MONTH:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getInt(pos));
		case BIGINT:
		case INTERVAL_DAY_TIME:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getLong(pos));
		case FLOAT:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getFloat(pos));
		case DOUBLE:
			return (csvMapper, container, array, pos) -> container.numberNode(array.getDouble(pos));
		case CHAR:
		case VARCHAR:
			return (csvMapper, container, array, pos) -> container.textNode(array.getString(pos).toString());
		case BINARY:
		case VARBINARY:
			return (csvMapper, container, array, pos) -> container.binaryNode(array.getBinary(pos));
		case DATE:
			return (csvMapper, container, array, pos) -> convertDate(array.getInt(pos), container);
		case TIME_WITHOUT_TIME_ZONE:
			return (csvMapper, container, array, pos) -> convertTime(array.getInt(pos), container);
		case TIMESTAMP_WITH_TIME_ZONE:
			final int zonedTimestampPrecision = ((LocalZonedTimestampType) fieldType).getPrecision();
			return (csvMapper, container, array, pos) ->
				convertTimestamp(array.getTimestamp(pos, zonedTimestampPrecision), container);
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			final int timestampPrecision = ((TimestampType) fieldType).getPrecision();
			return (csvMapper, container, array, pos) ->
				convertTimestamp(array.getTimestamp(pos, timestampPrecision), container);
		case DECIMAL:
			return createDecimalArrayElementConverter((DecimalType) fieldType);
		// we don't support ARRAY and ROW in an ARRAY, see CsvRowSchemaConverter#validateNestedField
		case ARRAY:
		case ROW:
		case MAP:
		case MULTISET:
		case RAW:
		default:
			throw new UnsupportedOperationException("Unsupported type: " + fieldType);
	}
}
 
Example #20
Source File: ParquetRowDataWriter.java    From flink with Apache License 2.0 4 votes vote down vote up
private FieldWriter createWriter(LogicalType t, Type type) {
	if (type.isPrimitive()) {
		switch (t.getTypeRoot()) {
			case CHAR:
			case VARCHAR:
				return new StringWriter();
			case BOOLEAN:
				return new BooleanWriter();
			case BINARY:
			case VARBINARY:
				return new BinaryWriter();
			case DECIMAL:
				DecimalType decimalType = (DecimalType) t;
				return createDecimalWriter(decimalType.getPrecision(), decimalType.getScale());
			case TINYINT:
				return new ByteWriter();
			case SMALLINT:
				return new ShortWriter();
			case DATE:
			case TIME_WITHOUT_TIME_ZONE:
			case INTEGER:
				return new IntWriter();
			case BIGINT:
				return new LongWriter();
			case FLOAT:
				return new FloatWriter();
			case DOUBLE:
				return new DoubleWriter();
			case TIMESTAMP_WITHOUT_TIME_ZONE:
				TimestampType timestampType = (TimestampType) t;
				return new TimestampWriter(timestampType.getPrecision());
			case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
				LocalZonedTimestampType localZonedTimestampType = (LocalZonedTimestampType) t;
				return new TimestampWriter(localZonedTimestampType.getPrecision());
			default:
				throw new UnsupportedOperationException("Unsupported type: " + type);
		}
	} else {
		throw new IllegalArgumentException("Unsupported  data type: " + t);
	}
}
 
Example #21
Source File: ArrayData.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the element object in the internal array data structure at the given position.
 *
 * @param array the internal array data
 * @param pos position of the element to return
 * @param elementType the element type of the array
 * @return the element object at the specified position in this array data
 * @deprecated Use {@link #createElementGetter(LogicalType)} for avoiding logical types during runtime.
 */
@Deprecated
static Object get(ArrayData array, int pos, LogicalType elementType) {
	if (array.isNullAt(pos)) {
		return null;
	}
	switch (elementType.getTypeRoot()) {
		case BOOLEAN:
			return array.getBoolean(pos);
		case TINYINT:
			return array.getByte(pos);
		case SMALLINT:
			return array.getShort(pos);
		case INTEGER:
		case DATE:
		case TIME_WITHOUT_TIME_ZONE:
		case INTERVAL_YEAR_MONTH:
			return array.getInt(pos);
		case BIGINT:
		case INTERVAL_DAY_TIME:
			return array.getLong(pos);
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			TimestampType timestampType = (TimestampType) elementType;
			return array.getTimestamp(pos, timestampType.getPrecision());
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			LocalZonedTimestampType lzTs = (LocalZonedTimestampType) elementType;
			return array.getTimestamp(pos, lzTs.getPrecision());
		case FLOAT:
			return array.getFloat(pos);
		case DOUBLE:
			return array.getDouble(pos);
		case CHAR:
		case VARCHAR:
			return array.getString(pos);
		case DECIMAL:
			DecimalType decimalType = (DecimalType) elementType;
			return array.getDecimal(pos, decimalType.getPrecision(), decimalType.getScale());
		case ARRAY:
			return array.getArray(pos);
		case MAP:
		case MULTISET:
			return array.getMap(pos);
		case ROW:
			return array.getRow(pos, ((RowType) elementType).getFieldCount());
		case STRUCTURED_TYPE:
			// not the most efficient code but ok for a deprecated method
			return array.getRow(pos, getFieldCount(elementType));
		case BINARY:
		case VARBINARY:
			return array.getBinary(pos);
		case RAW:
			return array.getRawValue(pos);
		default:
			throw new UnsupportedOperationException("Unsupported type: " + elementType);
	}
}
 
Example #22
Source File: ArrowUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
private static ArrowFieldWriter<RowData> createArrowFieldWriterForRow(ValueVector vector, LogicalType fieldType) {
	if (vector instanceof TinyIntVector) {
		return TinyIntWriter.forRow((TinyIntVector) vector);
	} else if (vector instanceof SmallIntVector) {
		return SmallIntWriter.forRow((SmallIntVector) vector);
	} else if (vector instanceof IntVector) {
		return IntWriter.forRow((IntVector) vector);
	} else if (vector instanceof BigIntVector) {
		return BigIntWriter.forRow((BigIntVector) vector);
	} else if (vector instanceof BitVector) {
		return BooleanWriter.forRow((BitVector) vector);
	} else if (vector instanceof Float4Vector) {
		return FloatWriter.forRow((Float4Vector) vector);
	} else if (vector instanceof Float8Vector) {
		return DoubleWriter.forRow((Float8Vector) vector);
	} else if (vector instanceof VarCharVector) {
		return VarCharWriter.forRow((VarCharVector) vector);
	} else if (vector instanceof VarBinaryVector) {
		return VarBinaryWriter.forRow((VarBinaryVector) vector);
	} else if (vector instanceof DecimalVector) {
		DecimalVector decimalVector = (DecimalVector) vector;
		return DecimalWriter.forRow(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
	} else if (vector instanceof DateDayVector) {
		return DateWriter.forRow((DateDayVector) vector);
	} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
		vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
		return TimeWriter.forRow(vector);
	} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
		int precision;
		if (fieldType instanceof LocalZonedTimestampType) {
			precision = ((LocalZonedTimestampType) fieldType).getPrecision();
		} else {
			precision = ((TimestampType) fieldType).getPrecision();
		}
		return TimestampWriter.forRow(vector, precision);
	} else if (vector instanceof ListVector) {
		ListVector listVector = (ListVector) vector;
		LogicalType elementType = ((ArrayType) fieldType).getElementType();
		return ArrayWriter.forRow(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
	} else if (vector instanceof StructVector) {
		RowType rowType = (RowType) fieldType;
		ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
		for (int i = 0; i < fieldsWriters.length; i++) {
			fieldsWriters[i] = createArrowFieldWriterForRow(
				((StructVector) vector).getVectorById(i),
				rowType.getTypeAt(i));
		}
		return RowWriter.forRow((StructVector) vector, fieldsWriters);
	} else {
		throw new UnsupportedOperationException(String.format(
			"Unsupported type %s.", fieldType));
	}
}
 
Example #23
Source File: ArrowUtils.java    From flink with Apache License 2.0 4 votes vote down vote up
private static ArrowFieldWriter<ArrayData> createArrowFieldWriterForArray(ValueVector vector, LogicalType fieldType) {
	if (vector instanceof TinyIntVector) {
		return TinyIntWriter.forArray((TinyIntVector) vector);
	} else if (vector instanceof SmallIntVector) {
		return SmallIntWriter.forArray((SmallIntVector) vector);
	} else if (vector instanceof IntVector) {
		return IntWriter.forArray((IntVector) vector);
	} else if (vector instanceof BigIntVector) {
		return BigIntWriter.forArray((BigIntVector) vector);
	} else if (vector instanceof BitVector) {
		return BooleanWriter.forArray((BitVector) vector);
	} else if (vector instanceof Float4Vector) {
		return FloatWriter.forArray((Float4Vector) vector);
	} else if (vector instanceof Float8Vector) {
		return DoubleWriter.forArray((Float8Vector) vector);
	} else if (vector instanceof VarCharVector) {
		return VarCharWriter.forArray((VarCharVector) vector);
	} else if (vector instanceof VarBinaryVector) {
		return VarBinaryWriter.forArray((VarBinaryVector) vector);
	} else if (vector instanceof DecimalVector) {
		DecimalVector decimalVector = (DecimalVector) vector;
		return DecimalWriter.forArray(decimalVector, getPrecision(decimalVector), decimalVector.getScale());
	} else if (vector instanceof DateDayVector) {
		return DateWriter.forArray((DateDayVector) vector);
	} else if (vector instanceof TimeSecVector || vector instanceof TimeMilliVector ||
		vector instanceof TimeMicroVector || vector instanceof TimeNanoVector) {
		return TimeWriter.forArray(vector);
	} else if (vector instanceof TimeStampVector && ((ArrowType.Timestamp) vector.getField().getType()).getTimezone() == null) {
		int precision;
		if (fieldType instanceof LocalZonedTimestampType) {
			precision = ((LocalZonedTimestampType) fieldType).getPrecision();
		} else {
			precision = ((TimestampType) fieldType).getPrecision();
		}
		return TimestampWriter.forArray(vector, precision);
	} else if (vector instanceof ListVector) {
		ListVector listVector = (ListVector) vector;
		LogicalType elementType = ((ArrayType) fieldType).getElementType();
		return ArrayWriter.forArray(listVector, createArrowFieldWriterForArray(listVector.getDataVector(), elementType));
	} else if (vector instanceof StructVector) {
		RowType rowType = (RowType) fieldType;
		ArrowFieldWriter<RowData>[] fieldsWriters = new ArrowFieldWriter[rowType.getFieldCount()];
		for (int i = 0; i < fieldsWriters.length; i++) {
			fieldsWriters[i] = createArrowFieldWriterForRow(
				((StructVector) vector).getVectorById(i),
				rowType.getTypeAt(i));
		}
		return RowWriter.forArray((StructVector) vector, fieldsWriters);
	} else {
		throw new UnsupportedOperationException(String.format(
			"Unsupported type %s.", fieldType));
	}
}
 
Example #24
Source File: ArrowUtilsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void init() {
	testFields = new ArrayList<>();
	testFields.add(Tuple7.of(
		"f1", new TinyIntType(), new ArrowType.Int(8, true), RowTinyIntWriter.class,
		TinyIntWriter.TinyIntWriterForRow.class, TinyIntFieldReader.class, ArrowTinyIntColumnVector.class));

	testFields.add(Tuple7.of("f2", new SmallIntType(), new ArrowType.Int(8 * 2, true),
		RowSmallIntWriter.class, SmallIntWriter.SmallIntWriterForRow.class, SmallIntFieldReader.class, ArrowSmallIntColumnVector.class));

	testFields.add(Tuple7.of("f3", new IntType(), new ArrowType.Int(8 * 4, true),
		RowIntWriter.class, IntWriter.IntWriterForRow.class, IntFieldReader.class, ArrowIntColumnVector.class));

	testFields.add(Tuple7.of("f4", new BigIntType(), new ArrowType.Int(8 * 8, true),
		RowBigIntWriter.class, BigIntWriter.BigIntWriterForRow.class, BigIntFieldReader.class, ArrowBigIntColumnVector.class));

	testFields.add(Tuple7.of("f5", new BooleanType(), new ArrowType.Bool(),
		RowBooleanWriter.class, BooleanWriter.BooleanWriterForRow.class, BooleanFieldReader.class, ArrowBooleanColumnVector.class));

	testFields.add(Tuple7.of("f6", new FloatType(), new ArrowType.FloatingPoint(FloatingPointPrecision.SINGLE),
		RowFloatWriter.class, FloatWriter.FloatWriterForRow.class, FloatFieldReader.class, ArrowFloatColumnVector.class));

	testFields.add(Tuple7.of("f7", new DoubleType(), new ArrowType.FloatingPoint(FloatingPointPrecision.DOUBLE),
		RowDoubleWriter.class, DoubleWriter.DoubleWriterForRow.class, DoubleFieldReader.class, ArrowDoubleColumnVector.class));

	testFields.add(Tuple7.of("f8", new VarCharType(), ArrowType.Utf8.INSTANCE,
		RowVarCharWriter.class, VarCharWriter.VarCharWriterForRow.class, VarCharFieldReader.class, ArrowVarCharColumnVector.class));

	testFields.add(Tuple7.of("f9", new VarBinaryType(), ArrowType.Binary.INSTANCE,
		RowVarBinaryWriter.class, VarBinaryWriter.VarBinaryWriterForRow.class, VarBinaryFieldReader.class, ArrowVarBinaryColumnVector.class));

	testFields.add(Tuple7.of("f10", new DecimalType(10, 3), new ArrowType.Decimal(10, 3),
		RowDecimalWriter.class, DecimalWriter.DecimalWriterForRow.class, DecimalFieldReader.class, ArrowDecimalColumnVector.class));

	testFields.add(Tuple7.of("f11", new DateType(), new ArrowType.Date(DateUnit.DAY),
		RowDateWriter.class, DateWriter.DateWriterForRow.class, DateFieldReader.class, ArrowDateColumnVector.class));

	testFields.add(Tuple7.of("f13", new TimeType(0), new ArrowType.Time(TimeUnit.SECOND, 32),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f14", new TimeType(2), new ArrowType.Time(TimeUnit.MILLISECOND, 32),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f15", new TimeType(4), new ArrowType.Time(TimeUnit.MICROSECOND, 64),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f16", new TimeType(8), new ArrowType.Time(TimeUnit.NANOSECOND, 64),
		RowTimeWriter.class, TimeWriter.TimeWriterForRow.class, TimeFieldReader.class, ArrowTimeColumnVector.class));

	testFields.add(Tuple7.of("f17", new LocalZonedTimestampType(0), new ArrowType.Timestamp(TimeUnit.SECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f18", new LocalZonedTimestampType(2), new ArrowType.Timestamp(TimeUnit.MILLISECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f19", new LocalZonedTimestampType(4), new ArrowType.Timestamp(TimeUnit.MICROSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f20", new LocalZonedTimestampType(8), new ArrowType.Timestamp(TimeUnit.NANOSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f21", new TimestampType(0), new ArrowType.Timestamp(TimeUnit.SECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f22", new TimestampType(2), new ArrowType.Timestamp(TimeUnit.MILLISECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f23", new TimestampType(4), new ArrowType.Timestamp(TimeUnit.MICROSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f24", new TimestampType(8), new ArrowType.Timestamp(TimeUnit.NANOSECOND, null),
		RowTimestampWriter.class, TimestampWriter.TimestampWriterForRow.class, TimestampFieldReader.class, ArrowTimestampColumnVector.class));

	testFields.add(Tuple7.of("f25", new ArrayType(new VarCharType()), ArrowType.List.INSTANCE,
		RowArrayWriter.class, ArrayWriter.ArrayWriterForRow.class, ArrayFieldReader.class, ArrowArrayColumnVector.class));

	RowType rowFieldType = new RowType(Arrays.asList(
		new RowType.RowField("a", new IntType()),
		new RowType.RowField("b", new VarCharType()),
		new RowType.RowField("c", new ArrayType(new VarCharType())),
		new RowType.RowField("d", new TimestampType(2)),
		new RowType.RowField("e", new RowType((Arrays.asList(
			new RowType.RowField("e1", new IntType()),
			new RowType.RowField("e2", new VarCharType())))))));
	testFields.add(Tuple7.of("f26", rowFieldType, ArrowType.Struct.INSTANCE,
		RowRowWriter.class, RowWriter.RowWriterForRow.class, RowFieldReader.class, ArrowRowColumnVector.class));

	List<RowType.RowField> rowFields = new ArrayList<>();
	for (Tuple7<String, LogicalType, ArrowType, Class<?>, Class<?>, Class<?>, Class<?>> field : testFields) {
		rowFields.add(new RowType.RowField(field.f0, field.f1));
	}
	rowType = new RowType(rowFields);

	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example #25
Source File: RowArrowReaderWriterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void init() {
	List<LogicalType> fieldTypes = new ArrayList<>();
	fieldTypes.add(new TinyIntType());
	fieldTypes.add(new SmallIntType());
	fieldTypes.add(new IntType());
	fieldTypes.add(new BigIntType());
	fieldTypes.add(new BooleanType());
	fieldTypes.add(new FloatType());
	fieldTypes.add(new DoubleType());
	fieldTypes.add(new VarCharType());
	fieldTypes.add(new VarBinaryType());
	fieldTypes.add(new DecimalType(10, 0));
	fieldTypes.add(new DateType());
	fieldTypes.add(new TimeType(0));
	fieldTypes.add(new TimeType(2));
	fieldTypes.add(new TimeType(4));
	fieldTypes.add(new TimeType(8));
	fieldTypes.add(new LocalZonedTimestampType(0));
	fieldTypes.add(new LocalZonedTimestampType(2));
	fieldTypes.add(new LocalZonedTimestampType(4));
	fieldTypes.add(new LocalZonedTimestampType(8));
	fieldTypes.add(new TimestampType(0));
	fieldTypes.add(new TimestampType(2));
	fieldTypes.add(new TimestampType(4));
	fieldTypes.add(new TimestampType(8));
	fieldTypes.add(new ArrayType(new VarCharType()));
	fieldTypes.add(new RowType(Arrays.asList(
		new RowType.RowField("a", new IntType()),
		new RowType.RowField("b", new VarCharType()),
		new RowType.RowField("c", new ArrayType(new VarCharType())),
		new RowType.RowField("d", new TimestampType(2)),
		new RowType.RowField("e", new RowType(Arrays.asList(
			new RowType.RowField("e1", new IntType()),
			new RowType.RowField("e2", new VarCharType())))))));

	List<RowType.RowField> rowFields = new ArrayList<>();
	for (int i = 0; i < fieldTypes.size(); i++) {
		rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
	}
	rowType = new RowType(rowFields);
	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example #26
Source File: RowDataArrowReaderWriterTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@BeforeClass
public static void init() {
	fieldTypes.add(new TinyIntType());
	fieldTypes.add(new SmallIntType());
	fieldTypes.add(new IntType());
	fieldTypes.add(new BigIntType());
	fieldTypes.add(new BooleanType());
	fieldTypes.add(new FloatType());
	fieldTypes.add(new DoubleType());
	fieldTypes.add(new VarCharType());
	fieldTypes.add(new VarBinaryType());
	fieldTypes.add(new DecimalType(10, 3));
	fieldTypes.add(new DateType());
	fieldTypes.add(new TimeType(0));
	fieldTypes.add(new TimeType(2));
	fieldTypes.add(new TimeType(4));
	fieldTypes.add(new TimeType(8));
	fieldTypes.add(new LocalZonedTimestampType(0));
	fieldTypes.add(new LocalZonedTimestampType(2));
	fieldTypes.add(new LocalZonedTimestampType(4));
	fieldTypes.add(new LocalZonedTimestampType(8));
	fieldTypes.add(new TimestampType(0));
	fieldTypes.add(new TimestampType(2));
	fieldTypes.add(new TimestampType(4));
	fieldTypes.add(new TimestampType(8));
	fieldTypes.add(new ArrayType(new VarCharType()));
	rowFieldType = new RowType(Arrays.asList(
		new RowType.RowField("a", new IntType()),
		new RowType.RowField("b", new VarCharType()),
		new RowType.RowField("c", new ArrayType(new VarCharType())),
		new RowType.RowField("d", new TimestampType(2)),
		new RowType.RowField("e", new RowType(Arrays.asList(
			new RowType.RowField("e1", new IntType()),
			new RowType.RowField("e2", new VarCharType()))))));
	fieldTypes.add(rowFieldType);

	List<RowType.RowField> rowFields = new ArrayList<>();
	for (int i = 0; i < fieldTypes.size(); i++) {
		rowFields.add(new RowType.RowField("f" + i, fieldTypes.get(i)));
	}
	rowType = new RowType(rowFields);
	allocator = ArrowUtils.getRootAllocator().newChildAllocator("stdout", 0, Long.MAX_VALUE);
}
 
Example #27
Source File: LogicalTypeDataTypeConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public DataType visit(LocalZonedTimestampType localZonedTimestampType) {
	return new AtomicDataType(localZonedTimestampType);
}
 
Example #28
Source File: LogicalTypeDataTypeConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public DataType visit(LocalZonedTimestampType localZonedTimestampType) {
	return new AtomicDataType(localZonedTimestampType);
}
 
Example #29
Source File: LogicalTypeChecks.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public Integer visit(LocalZonedTimestampType localZonedTimestampType) {
	return localZonedTimestampType.getPrecision();
}
 
Example #30
Source File: LogicalTypeChecks.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public TimestampKind visit(LocalZonedTimestampType localZonedTimestampType) {
	return localZonedTimestampType.getKind();
}