Java Code Examples for org.apache.flink.api.java.typeutils.TypeExtractor#createTypeInfo()

The following examples show how to use org.apache.flink.api.java.typeutils.TypeExtractor#createTypeInfo() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: CsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPojoTypeWithMappingInformation() throws Exception {
	File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
	tempFile.deleteOnExit();
	tempFile.setWritable(true);

	OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
	wrt.write("123,3.123,AAA,BBB\n");
	wrt.write("456,1.123,BBB,AAA\n");
	wrt.close();

	@SuppressWarnings("unchecked")
	PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
	CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[]{"field1", "field3", "field2", "field4"});

	inputFormat.configure(new Configuration());
	FileInputSplit[] splits = inputFormat.createInputSplits(1);

	inputFormat.open(splits[0]);

	validatePojoItem(inputFormat);
}
 
Example 2
Source Project: flink   File: UserDefinedFunctionHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tries to infer the TypeInformation of an AggregateFunction's accumulator type.
 *
 * @param tableFunction The TableFunction for which the accumulator type is inferred.
 * @param scalaType The implicitly inferred type of the accumulator type.
 * @return The inferred accumulator type of the AggregateFunction.
 */
public static <T> TypeInformation<T> getReturnTypeOfTableFunction(
		TableFunction<T> tableFunction,
		TypeInformation<T> scalaType) {

	TypeInformation<T> userProvidedType = tableFunction.getResultType();
	if (userProvidedType != null) {
		return userProvidedType;
	} else if (scalaType != null) {
		return scalaType;
	} else {
		return TypeExtractor.createTypeInfo(
			tableFunction,
			TableFunction.class,
			tableFunction.getClass(),
			0);
	}
}
 
Example 3
Source Project: sylph   File: StreamSqlUtil.java    License: Apache License 2.0 6 votes vote down vote up
private static TypeInformation<?> getFlinkType(Type type)
{
    if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == Map.class) {
        Type[] arguments = ((ParameterizedType) type).getActualTypeArguments();
        Type valueType = arguments[1];
        TypeInformation<?> valueInfo = getFlinkType(valueType);
        return new MapTypeInfo<>(TypeExtractor.createTypeInfo(arguments[0]), valueInfo);
    }
    else if (type instanceof ParameterizedType && ((ParameterizedType) type).getRawType() == List.class) {
        TypeInformation<?> typeInformation = getFlinkType(((ParameterizedType) type).getActualTypeArguments()[0]);
        if (typeInformation.isBasicType() && typeInformation != Types.STRING) {
            return Types.PRIMITIVE_ARRAY(typeInformation);
        }
        else {
            return Types.OBJECT_ARRAY(typeInformation);
        }
    }
    else {
        return TypeExtractor.createTypeInfo(type);
    }
}
 
Example 4
Source Project: Flink-CEPplus   File: Graph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Apply a function to the attribute of each vertex in the graph.
 *
 * @param mapper the map function to apply.
 * @return a new graph
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public <NV> Graph<K, NV, EV> mapVertices(final MapFunction<Vertex<K, VV>, NV> mapper) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) vertices.getType()).getTypeAt(0);

	TypeInformation<NV> valueType;

	if (mapper instanceof ResultTypeQueryable) {
		valueType = ((ResultTypeQueryable) mapper).getProducedType();
	} else {
		valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, vertices.getType(), null);
	}

	TypeInformation<Vertex<K, NV>> returnType = (TypeInformation<Vertex<K, NV>>) new TupleTypeInfo(
			Vertex.class, keyType, valueType);

	return mapVertices(mapper, returnType);
}
 
Example 5
Source Project: flink   File: StateDescriptor.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Create a new {@code StateDescriptor} with the given name and the given type information.
 *
 * <p>If this constructor fails (because it is not possible to describe the type via a class),
 * consider using the {@link #StateDescriptor(String, TypeInformation, Object)} constructor.
 *
 * @param name The name of the {@code StateDescriptor}.
 * @param type The class of the type of values in the state.
 * @param defaultValue The default value that will be set when requesting state without setting
 *                     a value before.
 */
protected StateDescriptor(String name, Class<T> type, @Nullable T defaultValue) {
	this.name = checkNotNull(name, "name must not be null");
	checkNotNull(type, "type class must not be null");

	try {
		this.typeInfo = TypeExtractor.createTypeInfo(type);
	} catch (Exception e) {
		throw new RuntimeException(
				"Could not create the type information for '" + type.getName() + "'. " +
				"The most common reason is failure to infer the generic type information, due to Java's type erasure. " +
				"In that case, please pass a 'TypeHint' instead of a class to describe the type. " +
				"For example, to describe 'Tuple2<String, String>' as a generic type, use " +
				"'new PravegaDeserializationSchema<>(new TypeHint<Tuple2<String, String>>(){}, serializer);'", e);
	}

	this.defaultValue = defaultValue;
}
 
Example 6
Source Project: flink   File: ExecutionEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Registers the given type with the serialization stack. If the type is eventually
 * serialized as a POJO, then the type is registered with the POJO serializer. If the
 * type ends up being serialized with Kryo, then it will be registered at Kryo to make
 * sure that only tags are written.
 *
 * @param type The class of the type to register.
 */
public void registerType(Class<?> type) {
	if (type == null) {
		throw new NullPointerException("Cannot register null type class.");
	}

	TypeInformation<?> typeInfo = TypeExtractor.createTypeInfo(type);

	if (typeInfo instanceof PojoTypeInfo) {
		config.registerPojoType(type);
	} else {
		config.registerKryoType(type);
	}
}
 
Example 7
Source Project: Flink-CEPplus   File: Graph.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a graph from a DataSet of edges.
 * Vertices are created automatically and their values are set
 * by applying the provided map function to the vertex IDs.
 *
 * @param edges a DataSet of edges.
 * @param vertexValueInitializer the mapper function that initializes the vertex values.
 * It allows to apply a map transformation on the vertex ID to produce an initial vertex value.
 * @param context the flink execution environment.
 * @return the newly created graph.
 */
public static <K, VV, EV> Graph<K, VV, EV> fromDataSet(DataSet<Edge<K, EV>> edges,
		final MapFunction<K, VV> vertexValueInitializer, ExecutionEnvironment context) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) edges.getType()).getTypeAt(0);

	TypeInformation<VV> valueType = TypeExtractor.createTypeInfo(
			MapFunction.class, vertexValueInitializer.getClass(), 1, keyType, null);

	@SuppressWarnings({ "unchecked", "rawtypes" })
	TypeInformation<Vertex<K, VV>> returnType = (TypeInformation<Vertex<K, VV>>) new TupleTypeInfo(
			Vertex.class, keyType, valueType);

	DataSet<Vertex<K, VV>> vertices = edges
		.flatMap(new EmitSrcAndTargetAsTuple1<>())
			.name("Source and target IDs")
		.distinct()
			.name("IDs")
		.map(new MapFunction<Tuple1<K>, Vertex<K, VV>>() {
			private Vertex<K, VV> output = new Vertex<>();

			public Vertex<K, VV> map(Tuple1<K> value) throws Exception {
				output.f0 = value.f0;
				output.f1 = vertexValueInitializer.map(value.f0);
				return output;
			}
		}).returns(returnType).withForwardedFields("f0").name("Initialize vertex values");

	return new Graph<>(vertices, edges, context);
}
 
Example 8
/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 
Example 9
Source Project: flink   File: TypeHint.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a hint for the generic type in the class signature.
 */
public TypeHint() {
	try {
		this.typeInfo = TypeExtractor.createTypeInfo(
				this, TypeHint.class, getClass(), 0);
	}
	catch (InvalidTypesException e) {
		throw new FlinkRuntimeException("The TypeHint is using a generic variable." +
				"This is not supported, generic types must be fully specified for the TypeHint.");
	}
}
 
Example 10
@Override
public TypeInformation<T> getProducedType() {
	if (this.getParser() instanceof ResultTypeQueryable) {
		return ((ResultTypeQueryable) this.getParser()).getProducedType();
	} else {
		return TypeExtractor.createTypeInfo(
			RowRecordParser.class,
			this.getParser().getClass(), 0, null, null);
	}
}
 
Example 11
Source Project: sylph   File: KafkaBaseSource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeInformation<Row> getProducedType()
{
    TypeInformation<?>[] types = new TypeInformation<?>[] {
            TypeExtractor.createTypeInfo(String.class),
            TypeExtractor.createTypeInfo(String.class), //createTypeInformation[String]
            TypeExtractor.createTypeInfo(String.class),
            Types.INT,
            Types.LONG
    };
    return new RowTypeInfo(types, KAFKA_COLUMNS);
}
 
Example 12
Source Project: flink   File: LegacyRowSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRowSerializerWithComplexTypes() {
	RowTypeInfo typeInfo = new RowTypeInfo(
		BasicTypeInfo.INT_TYPE_INFO,
		BasicTypeInfo.DOUBLE_TYPE_INFO,
		BasicTypeInfo.STRING_TYPE_INFO,
		new TupleTypeInfo<Tuple3<Integer, Boolean, Short>>(
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.BOOLEAN_TYPE_INFO,
			BasicTypeInfo.SHORT_TYPE_INFO),
		TypeExtractor.createTypeInfo(MyPojo.class));

	MyPojo testPojo1 = new MyPojo();
	testPojo1.name = null;
	MyPojo testPojo2 = new MyPojo();
	testPojo2.name = "Test1";
	MyPojo testPojo3 = new MyPojo();
	testPojo3.name = "Test2";

	Row[] data = new Row[]{
		createRow(null, null, null, null, null),
		createRow(0, null, null, null, null),
		createRow(0, 0.0, null, null, null),
		createRow(0, 0.0, "a", null, null),
		createRow(1, 0.0, "a", null, null),
		createRow(1, 1.0, "a", null, null),
		createRow(1, 1.0, "b", null, null),
		createRow(1, 1.0, "b", new Tuple3<>(1, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, false, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 2), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), null),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo1),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo2),
		createRow(1, 1.0, "b", new Tuple3<>(2, true, (short) 3), testPojo3)
	};

	TypeSerializer<Row> serializer = typeInfo.createLegacySerializer(new ExecutionConfig());
	RowSerializerTestInstance testInstance = new RowSerializerTestInstance(serializer, data);
	testInstance.testAll();
}
 
Example 13
@Test
public void testAtomicType() {
    StreamSchema<String> schema = new StreamSchema<>(TypeExtractor.createTypeInfo(String.class),
            "word");
    StreamSerializer<String> reader = new StreamSerializer<>(schema);
    Assert.assertArrayEquals(new Object[]{"Siddhi"}, reader.getRow("Siddhi"));
}
 
Example 14
Source Project: flink   File: MaxWithRetractAggFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public TypeInformation<MaxWithRetractAccumulator<T>> getAccumulatorType() {
	PojoTypeInfo pojoType = (PojoTypeInfo) TypeExtractor.createTypeInfo(MaxWithRetractAccumulator.class);
	List<PojoField> pojoFields = new ArrayList<>();
	for (int i = 0; i < pojoType.getTotalFields(); i++) {
		PojoField field = pojoType.getPojoFieldAt(i);
		if (field.getField().getName().equals("max")) {
			pojoFields.add(new PojoField(field.getField(), getValueTypeInfo()));
		} else {
			pojoFields.add(field);
		}
	}
	//noinspection unchecked
	return new PojoTypeInfo(pojoType.getTypeClass(), pojoFields);
}
 
Example 15
Source Project: flink   File: AvroRecordInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 
Example 16
Source Project: Flink-CEPplus   File: Graph.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeInformation<T> getProducedType() {
	return TypeExtractor.createTypeInfo(NeighborsFunction.class, function.getClass(), 3, null, null);
}
 
Example 17
@Test
public void testSerializerTree() {
	@SuppressWarnings("unchecked")
	TypeInformation<CollectionDataSets.PojoWithCollectionGeneric> ti =
			(TypeInformation<CollectionDataSets.PojoWithCollectionGeneric>)
					TypeExtractor.createTypeInfo(CollectionDataSets.PojoWithCollectionGeneric.class);

	String serTree = Utils.getSerializerTree(ti);
	// We can not test against the entire output because the fields of 'String' differ
	// between java versions
	Assert.assertTrue(serTree.startsWith("GenericTypeInfo (PojoWithCollectionGeneric)\n" +
			"    pojos:java.util.List\n" +
			"    key:int\n" +
			"    sqlDate:java.sql.Date\n" +
			"    bigInt:java.math.BigInteger\n" +
			"        signum:int\n" +
			"        mag:[I\n" +
			"        bitCount:int\n" +
			"        bitLength:int\n" +
			"        lowestSetBit:int\n" +
			"        firstNonzeroIntNum:int\n" +
			"    bigDecimalKeepItNull:java.math.BigDecimal\n" +
			"        intVal:java.math.BigInteger\n" +
			"            signum:int\n" +
			"            mag:[I\n" +
			"            bitCount:int\n" +
			"            bitLength:int\n" +
			"            lowestSetBit:int\n" +
			"            firstNonzeroIntNum:int\n" +
			"        scale:int\n" +
			"    scalaBigInt:scala.math.BigInt\n" +
			"        bigInteger:java.math.BigInteger\n" +
			"            signum:int\n" +
			"            mag:[I\n" +
			"            bitCount:int\n" +
			"            bitLength:int\n" +
			"            lowestSetBit:int\n" +
			"            firstNonzeroIntNum:int\n" +
			"    mixed:java.util.List\n" +
			"    makeMeGeneric:org.apache.flink.test.operators.util.CollectionDataSets$PojoWithDateAndEnum\n" +
			"        group:java.lang.String\n"));
}
 
Example 18
Source Project: flink   File: HadoopInputFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public TypeInformation<Tuple2<K, V>> getProducedType() {
	return new TupleTypeInfo<Tuple2<K, V>>(TypeExtractor.createTypeInfo(keyClass), TypeExtractor.createTypeInfo(valueClass));
}
 
Example 19
public FlinkMapperWrapper(DoFn<KV<KI, VI>, KV<KO, VO>> fn) {
	this.fn = fn;
	
	Type out = ((ParameterizedType) fn.getClass().getGenericSuperclass()).getActualTypeArguments()[1];
	Type kot = ((ParameterizedType) out).getActualTypeArguments()[0];
	Type vot = ((ParameterizedType) out).getActualTypeArguments()[1];
	
	
	TypeInformation<KO> koi = (TypeInformation<KO>) TypeExtractor.createTypeInfo(kot);
	TypeInformation<VO> voi = (TypeInformation<VO>) TypeExtractor.createTypeInfo(vot);
	
	typeInformation = new TupleTypeInfo<>(koi, voi);
}
 
Example 20
Source Project: flink   File: Types.java    License: Apache License 2.0 3 votes vote down vote up
/**
 * Returns type information for a POJO (Plain Old Java Object).
 *
 * <p>A POJO class is public and standalone (no non-static inner class). It has a public no-argument
 * constructor. All non-static, non-transient fields in the class (and all superclasses) are either public
 * (and non-final) or have a public getter and a setter method that follows the Java beans naming
 * conventions for getters and setters.
 *
 * <p>A POJO is a fixed-length and null-aware composite type. Every field can be null independent
 * of the field's type.
 *
 * <p>The generic types for all fields of the POJO can be defined in a hierarchy of subclasses.
 *
 * <p>If Flink's type analyzer is unable to extract a valid POJO type information with
 * type information for all fields, an {@link org.apache.flink.api.common.functions.InvalidTypesException}
 * is thrown. Alternatively, you can use {@link Types#POJO(Class, Map)} to specify all fields manually.
 *
 * @param pojoClass POJO class to be analyzed by Flink
 */
public static <T> TypeInformation<T> POJO(Class<T> pojoClass) {
	final TypeInformation<T> ti = TypeExtractor.createTypeInfo(pojoClass);
	if (ti instanceof PojoTypeInfo) {
		return ti;
	}
	throw new InvalidTypesException("POJO type expected but was: " + ti);
}