Java Code Examples for org.apache.flink.table.types.utils.TypeConversions#fromDataTypeToLegacyInfo()

/**
 * Retrieves all field accesses needed for the given {@link TimestampExtractor}.
 *
 * @param timestampExtractor Extractor for which to construct array of field accesses.
 * @param physicalInputType Physical input type that the timestamp extractor accesses.
 * @param nameRemapping Additional remapping of a logical to a physical field name.
 *                      TimestampExtractor works with logical names, but accesses physical
 *                      fields
 * @return Array of physical field references.
 */
public static ResolvedFieldReference[] getAccessedFields(
		TimestampExtractor timestampExtractor,
		DataType physicalInputType,
		Function<String, String> nameRemapping) {

	final Function<String, ResolvedFieldReference> fieldMapping;
	if (LogicalTypeChecks.isCompositeType(physicalInputType.getLogicalType())) {
		TableSchema schema = DataTypeUtils.expandCompositeTypeToSchema(physicalInputType);
		fieldMapping = (arg) -> mapToResolvedField(nameRemapping, schema, arg);
	} else {
		fieldMapping = (arg) -> new ResolvedFieldReference(
			arg,
			TypeConversions.fromDataTypeToLegacyInfo(physicalInputType),
			0);
	}
	return getAccessedFields(timestampExtractor, fieldMapping);
}
 

private static ResolvedFieldReference mapToResolvedField(
		Function<String, String> nameRemapping,
		TableSchema schema,
		String arg) {
	String remappedName = nameRemapping.apply(arg);

	int idx = IntStream.range(0, schema.getFieldCount())
		.filter(i -> schema.getFieldName(i).get().equals(remappedName))
		.findFirst()
		.orElseThrow(() -> new ValidationException(String.format("Field %s does not exist", remappedName)));

	TypeInformation<?> dataType = TypeConversions.fromDataTypeToLegacyInfo(schema.getTableColumn(idx)
		.get()
		.getType());
	return new ResolvedFieldReference(remappedName, dataType, idx);
}
 

/**
 * Extract result types for the aggregate or the table aggregate expression. For a table aggregate,
 * it may return multi result types when the composite return type is flattened.
 */
private Stream<DataType> extractAggregateResultTypes(ResolvedExpression expression) {
	if (ApiExpressionUtils.isFunctionOfKind(expression, TABLE_AGGREGATE)) {
		TypeInformation<?> legacyInfo = TypeConversions.fromDataTypeToLegacyInfo(expression.getOutputDataType());
		return Stream.of(FieldInfoUtils.getFieldTypes(legacyInfo))
			.map(TypeConversions::fromLegacyInfoToDataType);
	} else {
		return Stream.of(expression.getOutputDataType());
	}
}
 

@Test
public void testTableSourceUsingDescriptor() throws Exception {
    StreamExecutionEnvironment execEnvWrite = StreamExecutionEnvironment.getExecutionEnvironment();
    execEnvWrite.setParallelism(1);

    Stream stream = Stream.of(SETUP_UTILS.getScope(), "testJsonTableSource1");
    SETUP_UTILS.createTestStream(stream.getStreamName(), 1);

    // read data from the stream using Table reader
    TableSchema tableSchema = TableSchema.builder()
            .field("user", DataTypes.STRING())
            .field("uri", DataTypes.STRING())
            .field("accessTime", DataTypes.TIMESTAMP(3).bridgedTo(Timestamp.class))
            .build();
    TypeInformation<Row> typeInfo = (RowTypeInfo) TypeConversions.fromDataTypeToLegacyInfo(tableSchema.toRowDataType());

    PravegaConfig pravegaConfig = SETUP_UTILS.getPravegaConfig();

    // Write some data to the stream
    DataStreamSource<Row> dataStream = execEnvWrite
            .addSource(new TableEventSource(EVENT_COUNT_PER_SOURCE));

    FlinkPravegaWriter<Row> pravegaSink = FlinkPravegaWriter.<Row>builder()
            .withPravegaConfig(pravegaConfig)
            .forStream(stream)
            .withSerializationSchema(new JsonRowSerializationSchema.Builder(typeInfo).build())
            .withEventRouter((Row event) -> "fixedkey")
            .build();

    dataStream.addSink(pravegaSink);
    Assert.assertNotNull(execEnvWrite.getExecutionPlan());
    execEnvWrite.execute("PopulateRowData");

    testTableSourceStreamingDescriptor(stream, pravegaConfig);
    testTableSourceBatchDescriptor(stream, pravegaConfig);
}
 

@Test
public void testEmitDataStream() {
    FlinkPravegaWriter<Row> writer = mock(FlinkPravegaWriter.class);
    FlinkPravegaOutputFormat<Row> outputFormat = mock(FlinkPravegaOutputFormat.class);
    FlinkPravegaTableSink tableSink = new FlinkPravegaTableSink(schema -> writer, schema -> outputFormat, TUPLE1);
    TypeInformation<Row> rowTypeInfo = (TypeInformation<Row>) TypeConversions.fromDataTypeToLegacyInfo(TUPLE1.toRowDataType());
    DataStreamMock dataStream = new DataStreamMock(new StreamExecutionEnvironmentMock(), rowTypeInfo);
    tableSink.emitDataStream(dataStream);
    assertTrue(FlinkPravegaWriter.class.isAssignableFrom(dataStream.sinkFunction.getClass()));
}
 

/**
 * Extract result types for the aggregate or the table aggregate expression. For a table aggregate,
 * it may return multi result types when the composite return type is flattened.
 */
private Stream<DataType> extractAggregateResultTypes(ResolvedExpression expression) {
	if (ApiExpressionUtils.isFunctionOfKind(expression, TABLE_AGGREGATE)) {
		TypeInformation<?> legacyInfo = TypeConversions.fromDataTypeToLegacyInfo(expression.getOutputDataType());
		return Stream.of(FieldInfoUtils.getFieldTypes(legacyInfo))
			.map(TypeConversions::fromLegacyInfoToDataType);
	} else {
		return Stream.of(expression.getOutputDataType());
	}
}
 

@Override
@SuppressWarnings("unchecked")
public void open() throws Exception {
	super.open();
	this.cRowWrapper = new StreamRecordCRowWrappingCollector(output);
	CRowTypeInfo forwardedInputTypeInfo = new CRowTypeInfo(
		(RowTypeInfo) TypeConversions.fromDataTypeToLegacyInfo(
			TypeConversions.fromLogicalToDataType(inputType)));
	forwardedInputSerializer = forwardedInputTypeInfo.createSerializer(getExecutionConfig());
	udtfOutputTypeSerializer = PythonTypeUtils.toFlinkTypeSerializer(userDefinedFunctionOutputType);
}
 

@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);

	RowTypeInfo forwardedInputTypeInfo = (RowTypeInfo) TypeConversions.fromDataTypeToLegacyInfo(
		TypeConversions.fromLogicalToDataType(inputType));
	forwardedInputSerializer = forwardedInputTypeInfo.createSerializer(getRuntimeContext().getExecutionConfig());
}
 

public static TypeInformation<?> fromDataTypeToTypeInfo(DataType dataType) {
	Class<?> clazz = dataType.getConversionClass();
	if (clazz.isPrimitive()) {
		final TypeInformation<?> foundTypeInfo = primitiveDataTypeTypeInfoMap.get(clazz.getName());
		if (foundTypeInfo != null) {
			return foundTypeInfo;
		}
	}
	LogicalType logicalType = fromDataTypeToLogicalType(dataType);
	switch (logicalType.getTypeRoot()) {
		case DECIMAL:
			DecimalType decimalType = (DecimalType) logicalType;
			return clazz == Decimal.class ?
					new DecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale()) :
					new BigDecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
		case CHAR:
		case VARCHAR: // ignore precision
			return clazz == BinaryString.class ?
					BinaryStringTypeInfo.INSTANCE :
					BasicTypeInfo.STRING_TYPE_INFO;
		case BINARY:
		case VARBINARY: // ignore precision
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case INTERVAL_YEAR_MONTH:
			return TimeIntervalTypeInfo.INTERVAL_MONTHS;
		case INTERVAL_DAY_TIME:
			return TimeIntervalTypeInfo.INTERVAL_MILLIS;
		case ARRAY:
			if (dataType instanceof CollectionDataType &&
					!isPrimitive(((CollectionDataType) dataType).getElementDataType().getLogicalType())) {
				return ObjectArrayTypeInfo.getInfoFor(
						fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case MAP:
			KeyValueDataType mapType = (KeyValueDataType) dataType;
			return new MapTypeInfo(
					fromDataTypeToTypeInfo(mapType.getKeyDataType()),
					fromDataTypeToTypeInfo(mapType.getValueDataType()));
		case MULTISET:
			return MultisetTypeInfo.getInfoFor(
					fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
		case ROW:
			if (BaseRow.class.isAssignableFrom(dataType.getConversionClass())) {
				return BaseRowTypeInfo.of((RowType) fromDataTypeToLogicalType(dataType));
			} else if (Row.class == dataType.getConversionClass()) {
				FieldsDataType rowType = (FieldsDataType) dataType;
				RowType logicalRowType = (RowType) logicalType;
				return new RowTypeInfo(
						logicalRowType.getFieldNames().stream()
								.map(name -> rowType.getFieldDataTypes().get(name))
								.map(TypeInfoDataTypeConverter::fromDataTypeToTypeInfo)
								.toArray(TypeInformation[]::new),
						logicalRowType.getFieldNames().toArray(new String[0]));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		default:
			return TypeConversions.fromDataTypeToLegacyInfo(dataType);
	}
}
 

@SuppressWarnings("unchecked")
private TypeInformation<Row> getProducedTypeInformation() {
    return (TypeInformation<Row>) TypeConversions.fromDataTypeToLegacyInfo(getProducedDataType());
}
 

public static TypeInformation<?> fromDataTypeToTypeInfo(DataType dataType) {
	Class<?> clazz = dataType.getConversionClass();
	if (clazz.isPrimitive()) {
		final TypeInformation<?> foundTypeInfo = primitiveDataTypeTypeInfoMap.get(clazz.getName());
		if (foundTypeInfo != null) {
			return foundTypeInfo;
		}
	}
	LogicalType logicalType = fromDataTypeToLogicalType(dataType);
	switch (logicalType.getTypeRoot()) {
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			TimestampType timestampType = (TimestampType) logicalType;
			int precision = timestampType.getPrecision();
			if (timestampType.getKind() == TimestampKind.REGULAR) {
				return clazz == TimestampData.class ?
					new TimestampDataTypeInfo(precision) :
					(clazz == LocalDateTime.class ?
						((3 == precision) ?
							Types.LOCAL_DATE_TIME : new LegacyLocalDateTimeTypeInfo(precision)) :
						((3 == precision) ?
							Types.SQL_TIMESTAMP : new LegacyTimestampTypeInfo(precision)));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
			int precisionLzTs = lzTs.getPrecision();
			return clazz == TimestampData.class ?
				new TimestampDataTypeInfo(precisionLzTs) :
				(clazz == Instant.class ?
					((3 == precisionLzTs) ? Types.INSTANT : new LegacyInstantTypeInfo(precisionLzTs)) :
					TypeConversions.fromDataTypeToLegacyInfo(dataType));

		case DECIMAL:
			DecimalType decimalType = (DecimalType) logicalType;
			return clazz == DecimalData.class ?
					new DecimalDataTypeInfo(decimalType.getPrecision(), decimalType.getScale()) :
					new BigDecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
		case CHAR:
		case VARCHAR: // ignore precision
			return clazz == StringData.class ?
					StringDataTypeInfo.INSTANCE :
					BasicTypeInfo.STRING_TYPE_INFO;
		case BINARY:
		case VARBINARY: // ignore precision
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case INTERVAL_YEAR_MONTH:
			return TimeIntervalTypeInfo.INTERVAL_MONTHS;
		case INTERVAL_DAY_TIME:
			return TimeIntervalTypeInfo.INTERVAL_MILLIS;
		case ARRAY:
			if (dataType instanceof CollectionDataType &&
					!isPrimitive(((CollectionDataType) dataType).getElementDataType().getLogicalType())) {
				return ObjectArrayTypeInfo.getInfoFor(
						fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case MAP:
			KeyValueDataType mapType = (KeyValueDataType) dataType;
			return new MapTypeInfo(
					fromDataTypeToTypeInfo(mapType.getKeyDataType()),
					fromDataTypeToTypeInfo(mapType.getValueDataType()));
		case MULTISET:
			return MultisetTypeInfo.getInfoFor(
					fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
		case ROW:
			if (RowData.class.isAssignableFrom(dataType.getConversionClass())) {
				return RowDataTypeInfo.of((RowType) fromDataTypeToLogicalType(dataType));
			} else if (Row.class == dataType.getConversionClass()) {
				RowType logicalRowType = (RowType) logicalType;
				return new RowTypeInfo(
					dataType.getChildren()
						.stream()
						.map(TypeInfoDataTypeConverter::fromDataTypeToTypeInfo)
						.toArray(TypeInformation[]::new),
					logicalRowType.getFieldNames().toArray(new String[0]));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case RAW:
			if (logicalType instanceof RawType) {
				final RawType<?> rawType = (RawType<?>) logicalType;
				return createWrapperTypeInfo(rawType);
			}
			return TypeConversions.fromDataTypeToLegacyInfo(dataType);
		default:
			return TypeConversions.fromDataTypeToLegacyInfo(dataType);
	}
}
 

@Override
public TypeInformation<Row> getProducedType() {
	return (TypeInformation<Row>) TypeConversions.fromDataTypeToLegacyInfo(dataType);
}