org.apache.flink.api.java.typeutils.MapTypeInfo Java Examples

The following examples show how to use org.apache.flink.api.java.typeutils.MapTypeInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JsonRowDeserializationSchema.java    From flink with Apache License 2.0 6 votes vote down vote up
private Optional<DeserializationRuntimeConverter> createContainerConverter(TypeInformation<?> typeInfo) {
	if (typeInfo instanceof RowTypeInfo) {
		return Optional.of(createRowConverter((RowTypeInfo) typeInfo));
	} else if (typeInfo instanceof ObjectArrayTypeInfo) {
		return Optional.of(createObjectArrayConverter(((ObjectArrayTypeInfo) typeInfo).getComponentInfo()));
	} else if (typeInfo instanceof BasicArrayTypeInfo) {
		return Optional.of(createObjectArrayConverter(((BasicArrayTypeInfo) typeInfo).getComponentInfo()));
	} else if (isPrimitiveByteArray(typeInfo)) {
		return Optional.of(createByteArrayConverter());
	} else if (typeInfo instanceof MapTypeInfo) {
		MapTypeInfo<?, ?> mapTypeInfo = (MapTypeInfo<?, ?>) typeInfo;
		return Optional.of(createMapConverter(mapTypeInfo.getKeyTypeInfo(), mapTypeInfo.getValueTypeInfo()));
	} else {
		return Optional.empty();
	}
}
 
Example #2
Source File: StreamSqlUtil.java    From sylph with Apache License 2.0 6 votes vote down vote up
private static TypeInformation<?> getFlinkType(Type type)
{
    if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == Map.class) {
        Type[] arguments = ((ParameterizedType) type).getActualTypeArguments();
        Type valueType = arguments[1];
        TypeInformation<?> valueInfo = getFlinkType(valueType);
        return new MapTypeInfo<>(TypeExtractor.createTypeInfo(arguments[0]), valueInfo);
    }
    else if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == List.class) {
        TypeInformation<?> typeInformation = getFlinkType(((ParameterizedType) type).getActualTypeArguments()[0]);
        if (typeInformation.isBasicType() && typeInformation != Types.STRING) {
            return Types.PRIMITIVE_ARRAY(typeInformation);
        }
        else {
            return Types.OBJECT_ARRAY(typeInformation);
        }
    }
    else {
        return TypeExtractor.createTypeInfo(type);
    }
}
 
Example #3
Source File: JavaSqlITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testMap() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<Tuple2<Integer, Map<String, String>>> rows = new ArrayList<>();
	rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
	rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));

	TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
		BasicTypeInfo.INT_TYPE_INFO,
		new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));

	DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
	tableEnv.createTemporaryView("t1", ds1, $("a"), $("b"));

	String sqlQuery = "SELECT b['foo'] FROM t1";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "bar\n" + "spam\n";
	compareResultAsText(results, expected);
}
 
Example #4
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(List<Object> target, Object[] source, ObjectArrayTypeInfo objectArrayTypeInfo) {
	TypeInformation<?> itemType = objectArrayTypeInfo.getComponentInfo();
	for (Object field : source) {
		if (itemType instanceof RowTypeInfo) {
			Map<String, Object> nestedRow = new HashMap<>();
			convert(nestedRow, (Row) field,
				((RowTypeInfo) itemType).getFieldTypes(), ((RowTypeInfo) itemType).getFieldNames());
			target.add(nestedRow);
		} else if (itemType instanceof MapTypeInfo) {
			Map<String, Object> nestedMap = new HashMap<>();
			MapTypeInfo mapTypeInfo = (MapTypeInfo) itemType;
			convert(nestedMap, (Map<String, Object>) field, mapTypeInfo);
			target.add(nestedMap);
		} else if (itemType instanceof ObjectArrayTypeInfo) {
			List<Object> nestedObjectList = new ArrayList<>();
			convert(nestedObjectList, (Row[]) field, (ObjectArrayTypeInfo) itemType);
			target.add(nestedObjectList);
		}
	}

}
 
Example #5
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(Map<String, Object> target, Map<String, Object> source, MapTypeInfo mapTypeInfo) {
	TypeInformation valueTypeInfp = mapTypeInfo.getValueTypeInfo();

	for (String key : source.keySet()) {
		if (valueTypeInfp instanceof RowTypeInfo) {
			Map<String, Object> nestedRow = new HashMap<>();
			convert(nestedRow, (Row) source.get(key),
				((RowTypeInfo) valueTypeInfp).getFieldTypes(), ((RowTypeInfo) valueTypeInfp).getFieldNames());
			target.put(key, nestedRow);
		} else if (valueTypeInfp instanceof MapTypeInfo) {
			Map<String, Object> nestedMap = new HashMap<>();
			convert(nestedMap, (Map<String, Object>) source.get(key), (MapTypeInfo) valueTypeInfp);
			target.put(key, nestedMap);
		} else if (valueTypeInfp instanceof ObjectArrayTypeInfo) {
			List<Object> nestedObjectList = new ArrayList<>();
			convert(nestedObjectList, (Object[]) source.get(key), (ObjectArrayTypeInfo) valueTypeInfp);
			target.put(key, nestedObjectList);
		}
	}
}
 
Example #6
Source File: JavaSqlITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testMap() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<Tuple2<Integer, Map<String, String>>> rows = new ArrayList<>();
	rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
	rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));

	TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
		BasicTypeInfo.INT_TYPE_INFO,
		new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));

	DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
	tableEnv.registerDataSet("t1", ds1, "a, b");

	String sqlQuery = "SELECT b['foo'] FROM t1";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "bar\n" + "spam\n";
	compareResultAsText(results, expected);
}
 
Example #7
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(Map<String, Object> target, Map<String, Object> source, MapTypeInfo mapTypeInfo) {
	TypeInformation valueTypeInfp = mapTypeInfo.getValueTypeInfo();

	for (Map.Entry<String, Object> entry : source.entrySet()) {
		String key = entry.getKey();
		Object value = entry.getValue();
		if (valueTypeInfp instanceof RowTypeInfo) {
			Map<String, Object> nestedRow = new HashMap<>();
			convert(nestedRow, (Row) value,
				((RowTypeInfo) valueTypeInfp).getFieldTypes(), ((RowTypeInfo) valueTypeInfp).getFieldNames());
			target.put(key, nestedRow);
		} else if (valueTypeInfp instanceof MapTypeInfo) {
			Map<String, Object> nestedMap = new HashMap<>();
			convert(nestedMap, (Map<String, Object>) value, (MapTypeInfo) valueTypeInfp);
			target.put(key, nestedMap);
		} else if (valueTypeInfp instanceof ObjectArrayTypeInfo) {
			List<Object> nestedObjectList = new ArrayList<>();
			convert(nestedObjectList, (Object[]) value, (ObjectArrayTypeInfo) valueTypeInfp);
			target.put(key, nestedObjectList);
		}
	}
}
 
Example #8
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(List<Object> target, Object[] source, ObjectArrayTypeInfo objectArrayTypeInfo) {
	TypeInformation<?> itemType = objectArrayTypeInfo.getComponentInfo();
	for (Object field : source) {
		if (itemType instanceof RowTypeInfo) {
			Map<String, Object> nestedRow = new HashMap<>();
			convert(nestedRow, (Row) field,
				((RowTypeInfo) itemType).getFieldTypes(), ((RowTypeInfo) itemType).getFieldNames());
			target.add(nestedRow);
		} else if (itemType instanceof MapTypeInfo) {
			Map<String, Object> nestedMap = new HashMap<>();
			MapTypeInfo mapTypeInfo = (MapTypeInfo) itemType;
			convert(nestedMap, (Map<String, Object>) field, mapTypeInfo);
			target.add(nestedMap);
		} else if (itemType instanceof ObjectArrayTypeInfo) {
			List<Object> nestedObjectList = new ArrayList<>();
			convert(nestedObjectList, (Row[]) field, (ObjectArrayTypeInfo) itemType);
			target.add(nestedObjectList);
		}
	}

}
 
Example #9
Source File: JavaSqlITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testMap() throws Exception {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	BatchTableEnvironment tableEnv = BatchTableEnvironment.create(env, config());

	List<Tuple2<Integer, Map<String, String>>> rows = new ArrayList<>();
	rows.add(new Tuple2<>(1, Collections.singletonMap("foo", "bar")));
	rows.add(new Tuple2<>(2, Collections.singletonMap("foo", "spam")));

	TypeInformation<Tuple2<Integer, Map<String, String>>> ty = new TupleTypeInfo<>(
		BasicTypeInfo.INT_TYPE_INFO,
		new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));

	DataSet<Tuple2<Integer, Map<String, String>>> ds1 = env.fromCollection(rows, ty);
	tableEnv.registerDataSet("t1", ds1, "a, b");

	String sqlQuery = "SELECT b['foo'] FROM t1";
	Table result = tableEnv.sqlQuery(sqlQuery);

	DataSet<Row> resultSet = tableEnv.toDataSet(result, Row.class);
	List<Row> results = resultSet.collect();
	String expected = "bar\n" + "spam\n";
	compareResultAsText(results, expected);
}
 
Example #10
Source File: RowConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
MapKeyValueConverter(GroupType groupType, MapTypeInfo typeInfo) {
	this.keyConverter = createConverter(
		groupType.getType(0), 0, typeInfo.getKeyTypeInfo(),
		(fieldIndex, object) -> key = object);

	this.valueConverter = createConverter(
		groupType.getType(1), 1, typeInfo.getValueTypeInfo(),
		(fieldIndex, object) -> value = object);
}
 
Example #11
Source File: OrcTableSourceTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private TypeInformation[] getNestedFieldTypes() {
	return new TypeInformation[]{
		Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
		PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
		Types.ROW_NAMED(
			new String[]{"list"},
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		),
		ObjectArrayTypeInfo.getInfoFor(
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		),
		new MapTypeInfo<>(
			Types.STRING,
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		)
	};
}
 
Example #12
Source File: OrcTableSourceTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeInformation[] getNestedFieldTypes() {
	return new TypeInformation[]{
		Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
		PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
		Types.ROW_NAMED(
			new String[]{"list"},
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		),
		ObjectArrayTypeInfo.getInfoFor(
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		),
		new MapTypeInfo<>(
			Types.STRING,
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		)
	};
}
 
Example #13
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testProducedType() throws IOException {
	rowOrcInputFormat =
		new OrcRowInputFormat(getPath(TEST_FILE_NESTED), TEST_SCHEMA_NESTED, new Configuration());

	assertTrue(rowOrcInputFormat.getProducedType() instanceof RowTypeInfo);
	RowTypeInfo producedType = (RowTypeInfo) rowOrcInputFormat.getProducedType();

	assertArrayEquals(
		new TypeInformation[]{
			// primitives
			Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
			// binary
			PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
			// string
			Types.STRING,
			// struct
			Types.ROW_NAMED(
				new String[]{"list"},
				ObjectArrayTypeInfo.getInfoFor(
					Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))),
			// list
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING)),
			// map
			new MapTypeInfo<>(Types.STRING, Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))
		},
		producedType.getFieldTypes());
	assertArrayEquals(
		new String[]{"boolean1", "byte1", "short1", "int1", "long1", "float1", "double1", "bytes1", "string1", "middle", "list", "map"},
		producedType.getFieldNames());
}
 
Example #14
Source File: OrcTableSourceTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private TypeInformation[] getNestedFieldTypes() {
	return new TypeInformation[]{
		Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
		PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO, Types.STRING,
		Types.ROW_NAMED(
			new String[]{"list"},
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		),
		ObjectArrayTypeInfo.getInfoFor(
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		),
		new MapTypeInfo<>(
			Types.STRING,
			Types.ROW_NAMED(
				new String[]{"int1", "string1"},
				Types.INT, Types.STRING
			)
		)
	};
}
 
Example #15
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(Map<String, Object> map, Row row, TypeInformation<?>[] fieldTypes, String[] fieldNames) {
	for (int i = 0; i < fieldNames.length; i++) {
		if (row.getField(i) != null) {
			if (fieldTypes[i] instanceof BasicTypeInfo
				|| fieldTypes[i] instanceof PrimitiveArrayTypeInfo
				|| fieldTypes[i] instanceof BasicArrayTypeInfo) {
				map.put(fieldNames[i], row.getField(i));
			} else if (fieldTypes[i] instanceof RowTypeInfo) {
				Map<String, Object> nestedRow = new HashMap<>();
				RowTypeInfo nestedRowTypeInfo = (RowTypeInfo) fieldTypes[i];
				convert(nestedRow, (Row) row.getField(i),
					nestedRowTypeInfo.getFieldTypes(), nestedRowTypeInfo.getFieldNames());
				map.put(fieldNames[i], nestedRow);
			} else if (fieldTypes[i] instanceof MapTypeInfo) {
				Map<String, Object> nestedMap = new HashMap<>();
				MapTypeInfo mapTypeInfo = (MapTypeInfo) fieldTypes[i];
				convert(nestedMap, (Map<String, Object>) row.getField(i), mapTypeInfo);
				map.put(fieldNames[i], nestedMap);
			} else if (fieldTypes[i] instanceof ObjectArrayTypeInfo) {
				List<Object> nestedObjectList = new ArrayList<>();
				ObjectArrayTypeInfo objectArrayTypeInfo = (ObjectArrayTypeInfo) fieldTypes[i];
				convert(nestedObjectList, (Row[]) row.getField(i), objectArrayTypeInfo);
				map.put(fieldNames[i], nestedObjectList);
			}
		}
	}
}
 
Example #16
Source File: OrcRowInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testProducedType() throws IOException {
	rowOrcInputFormat =
		new OrcRowInputFormat(getPath(TEST_FILE_NESTED), TEST_SCHEMA_NESTED, new Configuration());

	assertTrue(rowOrcInputFormat.getProducedType() instanceof RowTypeInfo);
	RowTypeInfo producedType = (RowTypeInfo) rowOrcInputFormat.getProducedType();

	assertArrayEquals(
		new TypeInformation[]{
			// primitives
			Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
			// binary
			PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
			// string
			Types.STRING,
			// struct
			Types.ROW_NAMED(
				new String[]{"list"},
				ObjectArrayTypeInfo.getInfoFor(
					Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))),
			// list
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING)),
			// map
			new MapTypeInfo<>(Types.STRING, Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))
		},
		producedType.getFieldTypes());
	assertArrayEquals(
		new String[]{"boolean1", "byte1", "short1", "int1", "long1", "float1", "double1", "bytes1", "string1", "middle", "list", "map"},
		producedType.getFieldNames());
}
 
Example #17
Source File: OrcRowInputFormatTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testProducedType() throws IOException {
	rowOrcInputFormat =
		new OrcRowInputFormat(getPath(TEST_FILE_NESTED), TEST_SCHEMA_NESTED, new Configuration());

	assertTrue(rowOrcInputFormat.getProducedType() instanceof RowTypeInfo);
	RowTypeInfo producedType = (RowTypeInfo) rowOrcInputFormat.getProducedType();

	assertArrayEquals(
		new TypeInformation[]{
			// primitives
			Types.BOOLEAN, Types.BYTE, Types.SHORT, Types.INT, Types.LONG, Types.FLOAT, Types.DOUBLE,
			// binary
			PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO,
			// string
			Types.STRING,
			// struct
			Types.ROW_NAMED(
				new String[]{"list"},
				ObjectArrayTypeInfo.getInfoFor(
					Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))),
			// list
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING)),
			// map
			new MapTypeInfo<>(Types.STRING, Types.ROW_NAMED(new String[]{"int1", "string1"}, Types.INT, Types.STRING))
		},
		producedType.getFieldTypes());
	assertArrayEquals(
		new String[]{"boolean1", "byte1", "short1", "int1", "long1", "float1", "double1", "bytes1", "string1", "middle", "list", "map"},
		producedType.getFieldNames());
}
 
Example #18
Source File: LegacyTypeInfoDataTypeConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
public static DataType toDataType(TypeInformation<?> typeInfo) {
	// time indicators first as their hashCode/equals is shared with those of regular timestamps
	if (typeInfo instanceof TimeIndicatorTypeInfo) {
		return convertToTimeAttributeType((TimeIndicatorTypeInfo) typeInfo);
	}

	final DataType foundDataType = typeInfoDataTypeMap.get(typeInfo);
	if (foundDataType != null) {
		return foundDataType;
	}

	if (typeInfo instanceof RowTypeInfo) {
		return convertToRowType((RowTypeInfo) typeInfo);
	}

	else if (typeInfo instanceof ObjectArrayTypeInfo) {
		return convertToArrayType(
			typeInfo.getTypeClass(),
			((ObjectArrayTypeInfo) typeInfo).getComponentInfo());
	}

	else if (typeInfo instanceof BasicArrayTypeInfo) {
		return createLegacyType(LogicalTypeRoot.ARRAY, typeInfo);
	}

	else if (typeInfo instanceof MultisetTypeInfo) {
		return convertToMultisetType(((MultisetTypeInfo) typeInfo).getElementTypeInfo());
	}

	else if (typeInfo instanceof MapTypeInfo) {
		return convertToMapType((MapTypeInfo) typeInfo);
	}

	else if (typeInfo instanceof CompositeType) {
		return createLegacyType(LogicalTypeRoot.STRUCTURED_TYPE, typeInfo);
	}

	return createLegacyType(LogicalTypeRoot.ANY, typeInfo);
}
 
Example #19
Source File: RowConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
MapKeyValueConverter(GroupType groupType, MapTypeInfo typeInfo) {
	this.keyConverter = createConverter(
		groupType.getType(0), 0, typeInfo.getKeyTypeInfo(),
		(fieldIndex, object) -> key = object);

	this.valueConverter = createConverter(
		groupType.getType(1), 1, typeInfo.getValueTypeInfo(),
		(fieldIndex, object) -> value = object);
}
 
Example #20
Source File: LegacyTypeInfoDataTypeConverter.java    From flink with Apache License 2.0 5 votes vote down vote up
public static DataType toDataType(TypeInformation<?> typeInfo) {
	// time indicators first as their hashCode/equals is shared with those of regular timestamps
	if (typeInfo instanceof TimeIndicatorTypeInfo) {
		return convertToTimeAttributeType((TimeIndicatorTypeInfo) typeInfo);
	}

	final DataType foundDataType = typeInfoDataTypeMap.get(typeInfo);
	if (foundDataType != null) {
		return foundDataType;
	}

	if (typeInfo instanceof RowTypeInfo) {
		return convertToRowType((RowTypeInfo) typeInfo);
	}

	else if (typeInfo instanceof ObjectArrayTypeInfo) {
		return convertToArrayType(
			typeInfo.getTypeClass(),
			((ObjectArrayTypeInfo) typeInfo).getComponentInfo());
	}

	else if (typeInfo instanceof BasicArrayTypeInfo) {
		return createLegacyType(LogicalTypeRoot.ARRAY, typeInfo);
	}

	else if (typeInfo instanceof MultisetTypeInfo) {
		return convertToMultisetType(((MultisetTypeInfo) typeInfo).getElementTypeInfo());
	}

	else if (typeInfo instanceof MapTypeInfo) {
		return convertToMapType((MapTypeInfo) typeInfo);
	}

	else if (typeInfo instanceof CompositeType) {
		return createLegacyType(LogicalTypeRoot.STRUCTURED_TYPE, typeInfo);
	}

	return createLegacyType(LogicalTypeRoot.RAW, typeInfo);
}
 
Example #21
Source File: ParquetMapInputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void convert(Map<String, Object> map, Row row, TypeInformation<?>[] fieldTypes, String[] fieldNames) {
	for (int i = 0; i < fieldNames.length; i++) {
		if (row.getField(i) != null) {
			if (fieldTypes[i] instanceof BasicTypeInfo
				|| fieldTypes[i] instanceof PrimitiveArrayTypeInfo
				|| fieldTypes[i] instanceof BasicArrayTypeInfo) {
				map.put(fieldNames[i], row.getField(i));
			} else if (fieldTypes[i] instanceof RowTypeInfo) {
				Map<String, Object> nestedRow = new HashMap<>();
				RowTypeInfo nestedRowTypeInfo = (RowTypeInfo) fieldTypes[i];
				convert(nestedRow, (Row) row.getField(i),
					nestedRowTypeInfo.getFieldTypes(), nestedRowTypeInfo.getFieldNames());
				map.put(fieldNames[i], nestedRow);
			} else if (fieldTypes[i] instanceof MapTypeInfo) {
				Map<String, Object> nestedMap = new HashMap<>();
				MapTypeInfo mapTypeInfo = (MapTypeInfo) fieldTypes[i];
				convert(nestedMap, (Map<String, Object>) row.getField(i), mapTypeInfo);
				map.put(fieldNames[i], nestedMap);
			} else if (fieldTypes[i] instanceof ObjectArrayTypeInfo) {
				List<Object> nestedObjectList = new ArrayList<>();
				ObjectArrayTypeInfo objectArrayTypeInfo = (ObjectArrayTypeInfo) fieldTypes[i];
				convert(nestedObjectList, (Row[]) row.getField(i), objectArrayTypeInfo);
				map.put(fieldNames[i], nestedObjectList);
			}
		}
	}
}
 
Example #22
Source File: DataFormatConvertersTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testTypes() {
	for (int i = 0; i < simpleTypes.length; i++) {
		test(simpleTypes[i], simpleValues[i]);
	}
	test(new RowTypeInfo(simpleTypes), new Row(simpleTypes.length));
	test(new RowTypeInfo(simpleTypes), Row.ofKind(RowKind.DELETE, simpleValues));
	test(new RowDataTypeInfo(new VarCharType(VarCharType.MAX_LENGTH), new IntType()),
			GenericRowData.of(StringData.fromString("hehe"), 111));
	test(new RowDataTypeInfo(new VarCharType(VarCharType.MAX_LENGTH), new IntType()), GenericRowData.of(null, null));

	test(new DecimalDataTypeInfo(10, 5), null);
	test(new DecimalDataTypeInfo(10, 5), DecimalDataUtils.castFrom(5.555, 10, 5));

	test(Types.BIG_DEC, null);
	{
		DataFormatConverter converter = getConverter(Types.BIG_DEC);
		Assert.assertTrue(Arrays.deepEquals(
				new Object[]{converter.toInternal(converter.toExternal(DecimalDataUtils.castFrom(5, 19, 18)))},
				new Object[]{DecimalDataUtils.castFrom(5, 19, 18)}));
	}

	test(new ListTypeInfo<>(Types.STRING), null);
	test(new ListTypeInfo<>(Types.STRING), Arrays.asList("ahah", "xx"));

	test(BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, new Double[] {1D, 5D});
	test(BasicArrayTypeInfo.DOUBLE_ARRAY_TYPE_INFO, new Double[] {null, null});
	test(ObjectArrayTypeInfo.getInfoFor(Types.STRING), new String[] {null, null});
	test(ObjectArrayTypeInfo.getInfoFor(Types.STRING), new String[] {"haha", "hehe"});
	test(ObjectArrayTypeInfo.getInfoFor(Types.STRING), new String[] {"haha", "hehe"}, new String[] {"aa", "bb"});
	test(new MapTypeInfo<>(Types.STRING, Types.INT), null);

	HashMap<String, Integer> map = new HashMap<>();
	map.put("haha", 1);
	map.put("hah1", 5);
	map.put(null, null);
	test(new MapTypeInfo<>(Types.STRING, Types.INT), map);

	Tuple2 tuple2 = new Tuple2<>(5, 10);
	TupleTypeInfo tupleTypeInfo = new TupleTypeInfo<>(tuple2.getClass(), Types.INT, Types.INT);
	test(tupleTypeInfo, tuple2);

	test(TypeExtractor.createTypeInfo(MyPojo.class), new MyPojo(1, 3));
}
 
Example #23
Source File: LegacyTypeInfoDataTypeConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
private static DataType convertToMapType(MapTypeInfo typeInfo) {
	return DataTypes.MAP(
			toDataType(typeInfo.getKeyTypeInfo()),
			toDataType(typeInfo.getValueTypeInfo()))
		.bridgedTo(Map.class);
}
 
Example #24
Source File: TypeInfoDataTypeConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
public static TypeInformation<?> fromDataTypeToTypeInfo(DataType dataType) {
	Class<?> clazz = dataType.getConversionClass();
	if (clazz.isPrimitive()) {
		final TypeInformation<?> foundTypeInfo = primitiveDataTypeTypeInfoMap.get(clazz.getName());
		if (foundTypeInfo != null) {
			return foundTypeInfo;
		}
	}
	LogicalType logicalType = fromDataTypeToLogicalType(dataType);
	switch (logicalType.getTypeRoot()) {
		case TIMESTAMP_WITHOUT_TIME_ZONE:
			TimestampType timestampType = (TimestampType) logicalType;
			int precision = timestampType.getPrecision();
			if (timestampType.getKind() == TimestampKind.REGULAR) {
				return clazz == TimestampData.class ?
					new TimestampDataTypeInfo(precision) :
					(clazz == LocalDateTime.class ?
						((3 == precision) ?
							Types.LOCAL_DATE_TIME : new LegacyLocalDateTimeTypeInfo(precision)) :
						((3 == precision) ?
							Types.SQL_TIMESTAMP : new LegacyTimestampTypeInfo(precision)));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case TIMESTAMP_WITH_LOCAL_TIME_ZONE:
			LocalZonedTimestampType lzTs = (LocalZonedTimestampType) logicalType;
			int precisionLzTs = lzTs.getPrecision();
			return clazz == TimestampData.class ?
				new TimestampDataTypeInfo(precisionLzTs) :
				(clazz == Instant.class ?
					((3 == precisionLzTs) ? Types.INSTANT : new LegacyInstantTypeInfo(precisionLzTs)) :
					TypeConversions.fromDataTypeToLegacyInfo(dataType));

		case DECIMAL:
			DecimalType decimalType = (DecimalType) logicalType;
			return clazz == DecimalData.class ?
					new DecimalDataTypeInfo(decimalType.getPrecision(), decimalType.getScale()) :
					new BigDecimalTypeInfo(decimalType.getPrecision(), decimalType.getScale());
		case CHAR:
		case VARCHAR: // ignore precision
			return clazz == StringData.class ?
					StringDataTypeInfo.INSTANCE :
					BasicTypeInfo.STRING_TYPE_INFO;
		case BINARY:
		case VARBINARY: // ignore precision
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case INTERVAL_YEAR_MONTH:
			return TimeIntervalTypeInfo.INTERVAL_MONTHS;
		case INTERVAL_DAY_TIME:
			return TimeIntervalTypeInfo.INTERVAL_MILLIS;
		case ARRAY:
			if (dataType instanceof CollectionDataType &&
					!isPrimitive(((CollectionDataType) dataType).getElementDataType().getLogicalType())) {
				return ObjectArrayTypeInfo.getInfoFor(
						fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case MAP:
			KeyValueDataType mapType = (KeyValueDataType) dataType;
			return new MapTypeInfo(
					fromDataTypeToTypeInfo(mapType.getKeyDataType()),
					fromDataTypeToTypeInfo(mapType.getValueDataType()));
		case MULTISET:
			return MultisetTypeInfo.getInfoFor(
					fromDataTypeToTypeInfo(((CollectionDataType) dataType).getElementDataType()));
		case ROW:
			if (RowData.class.isAssignableFrom(dataType.getConversionClass())) {
				return RowDataTypeInfo.of((RowType) fromDataTypeToLogicalType(dataType));
			} else if (Row.class == dataType.getConversionClass()) {
				RowType logicalRowType = (RowType) logicalType;
				return new RowTypeInfo(
					dataType.getChildren()
						.stream()
						.map(TypeInfoDataTypeConverter::fromDataTypeToTypeInfo)
						.toArray(TypeInformation[]::new),
					logicalRowType.getFieldNames().toArray(new String[0]));
			} else {
				return TypeConversions.fromDataTypeToLegacyInfo(dataType);
			}
		case RAW:
			if (logicalType instanceof RawType) {
				final RawType<?> rawType = (RawType<?>) logicalType;
				return createWrapperTypeInfo(rawType);
			}
			return TypeConversions.fromDataTypeToLegacyInfo(dataType);
		default:
			return TypeConversions.fromDataTypeToLegacyInfo(dataType);
	}
}
 
Example #25
Source File: JdbcTypeUtilTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testTypeConversions() {
	assertEquals(Types.INTEGER, typeInformationToSqlType(BasicTypeInfo.INT_TYPE_INFO));
	testUnsupportedType(BasicTypeInfo.VOID_TYPE_INFO);
	testUnsupportedType(new MapTypeInfo<>(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO));
}
 
Example #26
Source File: OrcBatchReaderTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testNestedSchemaToTypeInfo1() {

	String schema =
		"struct<" +
			"middle:struct<" +
				"list:array<" +
					"struct<" +
						"int1:int," +
						"string1:string" +
					">" +
				">" +
			">," +
			"list:array<" +
				"struct<" +
					"int1:int," +
					"string1:string" +
				">" +
			">," +
			"map:map<" +
				"string," +
				"struct<" +
					"int1:int," +
					"string1:string" +
				">" +
			">" +
		">";
	TypeInformation typeInfo = OrcBatchReader.schemaToTypeInfo(TypeDescription.fromString(schema));

	Assert.assertNotNull(typeInfo);
	Assert.assertTrue(typeInfo instanceof RowTypeInfo);
	RowTypeInfo rowTypeInfo = (RowTypeInfo) typeInfo;

	// validate field types
	Assert.assertArrayEquals(
		new TypeInformation[]{
			Types.ROW_NAMED(
				new String[]{"list"},
				ObjectArrayTypeInfo.getInfoFor(
					Types.ROW_NAMED(
						new String[]{"int1", "string1"},
						Types.INT, Types.STRING
					)
				)
			),
			ObjectArrayTypeInfo.getInfoFor(
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			),
			new MapTypeInfo<>(
				Types.STRING,
				Types.ROW_NAMED(
					new String[]{"int1", "string1"},
					Types.INT, Types.STRING
				)
			)
		},
		rowTypeInfo.getFieldTypes());

	// validate field names
	Assert.assertArrayEquals(
		new String[] {"middle", "list", "map"},
		rowTypeInfo.getFieldNames());
}
 
Example #27
Source File: CollectAggFunction.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public TypeInformation<Map<T, Integer>> getResultType() {
	return new MapTypeInfo<>(elementType, Types.INT);
}
 
Example #28
Source File: OrcBatchReader.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 
Example #29
Source File: OrcBatchReader.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Converts an ORC schema to a Flink TypeInformation.
 *
 * @param schema The ORC schema.
 * @return The TypeInformation that corresponds to the ORC schema.
 */
static TypeInformation schemaToTypeInfo(TypeDescription schema) {
	switch (schema.getCategory()) {
		case BOOLEAN:
			return BasicTypeInfo.BOOLEAN_TYPE_INFO;
		case BYTE:
			return BasicTypeInfo.BYTE_TYPE_INFO;
		case SHORT:
			return BasicTypeInfo.SHORT_TYPE_INFO;
		case INT:
			return BasicTypeInfo.INT_TYPE_INFO;
		case LONG:
			return BasicTypeInfo.LONG_TYPE_INFO;
		case FLOAT:
			return BasicTypeInfo.FLOAT_TYPE_INFO;
		case DOUBLE:
			return BasicTypeInfo.DOUBLE_TYPE_INFO;
		case DECIMAL:
			return BasicTypeInfo.BIG_DEC_TYPE_INFO;
		case STRING:
		case CHAR:
		case VARCHAR:
			return BasicTypeInfo.STRING_TYPE_INFO;
		case DATE:
			return SqlTimeTypeInfo.DATE;
		case TIMESTAMP:
			return SqlTimeTypeInfo.TIMESTAMP;
		case BINARY:
			return PrimitiveArrayTypeInfo.BYTE_PRIMITIVE_ARRAY_TYPE_INFO;
		case STRUCT:
			List<TypeDescription> fieldSchemas = schema.getChildren();
			TypeInformation[] fieldTypes = new TypeInformation[fieldSchemas.size()];
			for (int i = 0; i < fieldSchemas.size(); i++) {
				fieldTypes[i] = schemaToTypeInfo(fieldSchemas.get(i));
			}
			String[] fieldNames = schema.getFieldNames().toArray(new String[]{});
			return new RowTypeInfo(fieldTypes, fieldNames);
		case LIST:
			TypeDescription elementSchema = schema.getChildren().get(0);
			TypeInformation<?> elementType = schemaToTypeInfo(elementSchema);
			// arrays of primitive types are handled as object arrays to support null values
			return ObjectArrayTypeInfo.getInfoFor(elementType);
		case MAP:
			TypeDescription keySchema = schema.getChildren().get(0);
			TypeDescription valSchema = schema.getChildren().get(1);
			TypeInformation<?> keyType = schemaToTypeInfo(keySchema);
			TypeInformation<?> valType = schemaToTypeInfo(valSchema);
			return new MapTypeInfo<>(keyType, valType);
		case UNION:
			throw new UnsupportedOperationException("UNION type is not supported yet.");
		default:
			throw new IllegalArgumentException("Unknown type " + schema);
	}
}
 
Example #30
Source File: RowConverter.java    From flink with Apache License 2.0 4 votes vote down vote up
MapConverter(GroupType type, MapTypeInfo typeInfo, ParentDataHolder parentDataHolder, int pos) {
	this.parentDataHolder = parentDataHolder;
	this.pos = pos;
	this.keyValueConverter = new MapKeyValueConverter((GroupType) type.getType(0), typeInfo);
}