Java Code Examples for org.apache.flink.table.api.DataTypes#MAP
The following examples show how to use
org.apache.flink.table.api.DataTypes#MAP .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: HiveCatalogDataTypeTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testComplexDataTypes() throws Exception {
DataType[] types = new DataType[]{
DataTypes.ARRAY(DataTypes.DOUBLE()),
DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT()),
DataTypes.ROW(
DataTypes.FIELD("0", DataTypes.BOOLEAN()),
DataTypes.FIELD("1", DataTypes.BOOLEAN()),
DataTypes.FIELD("2", DataTypes.DATE())),
// nested complex types
DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.INT())),
DataTypes.MAP(DataTypes.STRING(), DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())),
DataTypes.ROW(
DataTypes.FIELD("3", DataTypes.ARRAY(DataTypes.DECIMAL(5, 3))),
DataTypes.FIELD("4", DataTypes.MAP(DataTypes.TINYINT(), DataTypes.SMALLINT())),
DataTypes.FIELD("5", DataTypes.ROW(DataTypes.FIELD("3", DataTypes.TIMESTAMP())))
)
};
verifyDataTypes(types);
}
Example 2
| Source File: HiveCatalogDataTypeTest.java From flink with Apache License 2.0 | 6 votes |
|
@Test
public void testComplexDataTypes() throws Exception {
DataType[] types = new DataType[]{
DataTypes.ARRAY(DataTypes.DOUBLE()),
DataTypes.MAP(DataTypes.FLOAT(), DataTypes.BIGINT()),
DataTypes.ROW(
DataTypes.FIELD("0", DataTypes.BOOLEAN()),
DataTypes.FIELD("1", DataTypes.BOOLEAN()),
DataTypes.FIELD("2", DataTypes.DATE())),
// nested complex types
DataTypes.ARRAY(DataTypes.ARRAY(DataTypes.INT())),
DataTypes.MAP(DataTypes.STRING(), DataTypes.MAP(DataTypes.STRING(), DataTypes.BIGINT())),
DataTypes.ROW(
DataTypes.FIELD("3", DataTypes.ARRAY(DataTypes.DECIMAL(5, 3))),
DataTypes.FIELD("4", DataTypes.MAP(DataTypes.TINYINT(), DataTypes.SMALLINT())),
DataTypes.FIELD("5", DataTypes.ROW(DataTypes.FIELD("3", DataTypes.TIMESTAMP(9))))
)
};
verifyDataTypes(types);
}
Example 3
| Source File: DataTypeExtractor.java From flink with Apache License 2.0 | 6 votes |
|
private @Nullable DataType extractMapType(DataTypeTemplate template, List<Type> typeHierarchy, Type type) {
final Class<?> clazz = toClass(type);
// we only allow Map here (not a subclass) because we cannot guarantee more specific
// data structures after conversion
if (clazz != Map.class) {
return null;
}
if (!(type instanceof ParameterizedType)) {
throw extractionError(
"The class '%s' needs generic parameters for a map type.",
Map.class.getName());
}
final ParameterizedType parameterizedType = (ParameterizedType) type;
final DataType key = extractDataTypeOrRaw(
template,
typeHierarchy,
parameterizedType.getActualTypeArguments()[0]);
final DataType value = extractDataTypeOrRaw(
template,
typeHierarchy,
parameterizedType.getActualTypeArguments()[1]);
return DataTypes.MAP(key, value);
}
Example 4
| Source File: HiveTypeUtil.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Convert Hive data type to a Flink data type.
*
* @param hiveType a Hive data type
* @return the corresponding Flink data type
*/
public static DataType toFlinkType(TypeInfo hiveType) {
checkNotNull(hiveType, "hiveType cannot be null");
switch (hiveType.getCategory()) {
case PRIMITIVE:
return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
case LIST:
ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;
List<String> names = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();
DataTypes.Field[] fields = new DataTypes.Field[names.size()];
for (int i = 0; i < fields.length; i++) {
fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
}
return DataTypes.ROW(fields);
default:
throw new UnsupportedOperationException(
String.format("Flink doesn't support Hive data type %s yet.", hiveType));
}
}
Example 5
| Source File: HiveTypeUtil.java From flink with Apache License 2.0 | 5 votes |
|
/**
* Convert Hive data type to a Flink data type.
*
* @param hiveType a Hive data type
* @return the corresponding Flink data type
*/
public static DataType toFlinkType(TypeInfo hiveType) {
checkNotNull(hiveType, "hiveType cannot be null");
switch (hiveType.getCategory()) {
case PRIMITIVE:
return toFlinkPrimitiveType((PrimitiveTypeInfo) hiveType);
case LIST:
ListTypeInfo listTypeInfo = (ListTypeInfo) hiveType;
return DataTypes.ARRAY(toFlinkType(listTypeInfo.getListElementTypeInfo()));
case MAP:
MapTypeInfo mapTypeInfo = (MapTypeInfo) hiveType;
return DataTypes.MAP(toFlinkType(mapTypeInfo.getMapKeyTypeInfo()), toFlinkType(mapTypeInfo.getMapValueTypeInfo()));
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) hiveType;
List<String> names = structTypeInfo.getAllStructFieldNames();
List<TypeInfo> typeInfos = structTypeInfo.getAllStructFieldTypeInfos();
DataTypes.Field[] fields = new DataTypes.Field[names.size()];
for (int i = 0; i < fields.length; i++) {
fields[i] = DataTypes.FIELD(names.get(i), toFlinkType(typeInfos.get(i)));
}
return DataTypes.ROW(fields);
default:
throw new UnsupportedOperationException(
String.format("Flink doesn't support Hive data type %s yet.", hiveType));
}
}
Example 6
| Source File: SchemaUtils.java From pulsar-flink with Apache License 2.0 | 4 votes |
|
private static DataType avro2SqlType(Schema avroSchema, Set<String> existingRecordNames) throws IncompatibleSchemaException {
LogicalType logicalType = avroSchema.getLogicalType();
switch (avroSchema.getType()) {
case INT:
if (logicalType instanceof LogicalTypes.Date) {
return DataTypes.DATE();
} else {
return DataTypes.INT();
}
case STRING:
case ENUM:
return DataTypes.STRING();
case BOOLEAN:
return DataTypes.BOOLEAN();
case BYTES:
case FIXED:
// For FIXED type, if the precision requires more bytes than fixed size, the logical
// type will be null, which is handled by Avro library.
if (logicalType instanceof LogicalTypes.Decimal) {
LogicalTypes.Decimal d = (LogicalTypes.Decimal) logicalType;
return DataTypes.DECIMAL(d.getPrecision(), d.getScale());
} else {
return DataTypes.BYTES();
}
case DOUBLE:
return DataTypes.DOUBLE();
case FLOAT:
return DataTypes.FLOAT();
case LONG:
if (logicalType instanceof LogicalTypes.TimestampMillis ||
logicalType instanceof LogicalTypes.TimestampMicros) {
return DataTypes.TIMESTAMP(3).bridgedTo(java.sql.Timestamp.class);
} else {
return DataTypes.BIGINT();
}
case RECORD:
if (existingRecordNames.contains(avroSchema.getFullName())) {
throw new IncompatibleSchemaException(
String.format("Found recursive reference in Avro schema, which can not be processed by Flink: %s", avroSchema.toString(true)), null);
}
Set<String> newRecordName = ImmutableSet.<String>builder()
.addAll(existingRecordNames).add(avroSchema.getFullName()).build();
List<DataTypes.Field> fields = new ArrayList<>();
for (Schema.Field f : avroSchema.getFields()) {
DataType fieldType = avro2SqlType(f.schema(), newRecordName);
fields.add(DataTypes.FIELD(f.name(), fieldType));
}
return DataTypes.ROW(fields.toArray(new DataTypes.Field[0]));
case ARRAY:
DataType elementType = avro2SqlType(avroSchema.getElementType(), existingRecordNames);
return DataTypes.ARRAY(elementType);
case MAP:
DataType valueType = avro2SqlType(avroSchema.getValueType(), existingRecordNames);
return DataTypes.MAP(DataTypes.STRING(), valueType);
case UNION:
if (avroSchema.getTypes().stream().anyMatch(f -> f.getType() == Schema.Type.NULL)) {
// In case of a union with null, eliminate it and make a recursive call
List<Schema> remainingUnionTypes =
avroSchema.getTypes().stream().filter(f -> f.getType() != Schema.Type.NULL).collect(Collectors.toList());
if (remainingUnionTypes.size() == 1) {
return avro2SqlType(remainingUnionTypes.get(0), existingRecordNames).nullable();
} else {
return avro2SqlType(Schema.createUnion(remainingUnionTypes), existingRecordNames).nullable();
}
} else {
List<Schema.Type> types = avroSchema.getTypes().stream().map(Schema::getType).collect(Collectors.toList());
if (types.size() == 1) {
return avro2SqlType(avroSchema.getTypes().get(0), existingRecordNames);
} else if (types.size() == 2 && types.contains(Schema.Type.INT) && types.contains(Schema.Type.LONG)) {
return DataTypes.BIGINT();
} else if (types.size() == 2 && types.contains(Schema.Type.FLOAT) && types.contains(Schema.Type.DOUBLE)) {
return DataTypes.DOUBLE();
} else {
// Convert complex unions to struct types where field names are member0, member1, etc.
// This is consistent with the behavior when converting between Avro and Parquet.
List<DataTypes.Field> memberFields = new ArrayList<>();
List<Schema> schemas = avroSchema.getTypes();
for (int i = 0; i < schemas.size(); i++) {
DataType memberType = avro2SqlType(schemas.get(i), existingRecordNames);
memberFields.add(DataTypes.FIELD("member" + i, memberType));
}
return DataTypes.ROW(memberFields.toArray(new DataTypes.Field[0]));
}
}
default:
throw new IncompatibleSchemaException(String.format("Unsupported type %s", avroSchema.toString(true)), null);
}
}
