Java Code Examples for org.apache.flink.api.java.typeutils.TypeExtractor

The following examples show how to use org.apache.flink.api.java.typeutils.TypeExtractor. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: ExpressionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidPojo() throws Throwable {
	TypeInformation<ComplexNestedClass> ti = TypeExtractor.getForClass(ComplexNestedClass.class);

	String[][] tests = new String[][] {
		new String[] {"nonexistent"},
		new String[] {"date.abc"} // nesting into unnested
	};
	for (String[] test : tests) {
		Throwable e = null;
		try {
			new ExpressionKeys<>(test, ti);
		} catch (Throwable t) {
			e = t;
		}
		Assert.assertNotNull(e);
	}
}
 
Example 2
Source Project: Flink-CEPplus   Source File: Graph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Apply a function to the attribute of each vertex in the graph.
 *
 * @param mapper the map function to apply.
 * @return a new graph
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public <NV> Graph<K, NV, EV> mapVertices(final MapFunction<Vertex<K, VV>, NV> mapper) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) vertices.getType()).getTypeAt(0);

	TypeInformation<NV> valueType;

	if (mapper instanceof ResultTypeQueryable) {
		valueType = ((ResultTypeQueryable) mapper).getProducedType();
	} else {
		valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, vertices.getType(), null);
	}

	TypeInformation<Vertex<K, NV>> returnType = (TypeInformation<Vertex<K, NV>>) new TupleTypeInfo(
			Vertex.class, keyType, valueType);

	return mapVertices(mapper, returnType);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: Graph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Apply a function to the attribute of each edge in the graph.
 *
 * @param mapper the map function to apply.
 * @return a new graph
 */
@SuppressWarnings({ "unchecked", "rawtypes" })
public <NV> Graph<K, VV, NV> mapEdges(final MapFunction<Edge<K, EV>, NV> mapper) {

	TypeInformation<K> keyType = ((TupleTypeInfo<?>) edges.getType()).getTypeAt(0);

	TypeInformation<NV> valueType;

	if (mapper instanceof ResultTypeQueryable) {
		valueType = ((ResultTypeQueryable) mapper).getProducedType();
	} else {
		valueType = TypeExtractor.createTypeInfo(MapFunction.class, mapper.getClass(), 1, edges.getType(), null);
	}

	TypeInformation<Edge<K, NV>> returnType = (TypeInformation<Edge<K, NV>>) new TupleTypeInfo(
			Edge.class, keyType, keyType, valueType);

	return mapEdges(mapper, returnType);
}
 
Example 4
Source Project: flink   Source File: WindowedStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given window function to each window. The window function is called for each
 * evaluation of the window for each key individually. The output of the window function is
 * interpreted as a regular non-windowed stream.
 *
 * <p>Arriving data is incrementally aggregated using the given aggregate function. This means
 * that the window function typically has only a single value to process when called.
 *
 * @param aggFunction The aggregate function that is used for incremental aggregation.
 * @param windowFunction The window function.
 *
 * @return The data stream that is the result of applying the window function to the window.
 *
 * @param <ACC> The type of the AggregateFunction's accumulator
 * @param <V> The type of AggregateFunction's result, and the WindowFunction's input
 * @param <R> The type of the elements in the resulting stream, equal to the
 *            WindowFunction's result type
 */
@PublicEvolving
public <ACC, V, R> SingleOutputStreamOperator<R> aggregate(
		AggregateFunction<T, ACC, V> aggFunction,
		ProcessWindowFunction<V, R, K, W> windowFunction) {

	checkNotNull(aggFunction, "aggFunction");
	checkNotNull(windowFunction, "windowFunction");

	TypeInformation<ACC> accumulatorType = TypeExtractor.getAggregateFunctionAccumulatorType(
			aggFunction, input.getType(), null, false);

	TypeInformation<V> aggResultType = TypeExtractor.getAggregateFunctionReturnType(
			aggFunction, input.getType(), null, false);

	TypeInformation<R> resultType = getProcessWindowFunctionReturnType(windowFunction, aggResultType, null);

	return aggregate(aggFunction, windowFunction, accumulatorType, aggResultType, resultType);
}
 
Example 5
Source Project: flink   Source File: ConnectedStreams.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given {@link CoProcessFunction} on the connected input streams,
 * thereby creating a transformed output stream.
 *
 * <p>The function will be called for every element in the input streams and can produce zero or
 * more output elements. Contrary to the {@link #flatMap(CoFlatMapFunction)} function, this
 * function can also query the time and set timers. When reacting to the firing of set timers
 * the function can directly emit elements and/or register yet more timers.
 *
 * @param coProcessFunction The {@link CoProcessFunction} that is called for each element
 *                      in the stream.
 *
 * @param <R> The type of elements emitted by the {@code CoProcessFunction}.
 *
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <R> SingleOutputStreamOperator<R> process(
		CoProcessFunction<IN1, IN2, R> coProcessFunction) {

	TypeInformation<R> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
		coProcessFunction,
		CoProcessFunction.class,
		0,
		1,
		2,
		TypeExtractor.NO_INDEX,
		getType1(),
		getType2(),
		Utils.getCallLocationName(),
		true);

	return process(coProcessFunction, outTypeInfo);
}
 
Example 6
Source Project: flink   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: ExecutionEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
public <X> DataSource<X> readFile(FileInputFormat<X> inputFormat, String filePath) {
	if (inputFormat == null) {
		throw new IllegalArgumentException("InputFormat must not be null.");
	}
	if (filePath == null) {
		throw new IllegalArgumentException("The file path must not be null.");
	}

	inputFormat.setFilePath(new Path(filePath));
	try {
		return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
	}
	catch (Exception e) {
		throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +
				"Please specify the TypeInformation of the produced type explicitly by using the " +
				"'createInput(InputFormat, TypeInformation)' method instead.");
	}
}
 
Example 8
Source Project: flink   Source File: SelectorFunctionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAreCompatible1() throws Keys.IncompatibleKeysException {
	TypeInformation<Pojo2> t1 = TypeExtractor.getForClass(Pojo2.class);
	TypeInformation<Tuple2<Integer, String>> t2 =
		new TupleTypeInfo<>(BasicTypeInfo.INT_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO);

	Keys<Pojo2> k1 = new Keys.SelectorFunctionKeys<>(
		new KeySelector1(),
		t1,
		BasicTypeInfo.STRING_TYPE_INFO
	);
	Keys<Tuple2<Integer, String>> k2 = new Keys.SelectorFunctionKeys<>(
		new KeySelector2(),
		t2,
		BasicTypeInfo.STRING_TYPE_INFO
	);

	Assert.assertTrue(k1.areCompatible(k2));
	Assert.assertTrue(k2.areCompatible(k1));
}
 
Example 9
Source Project: flink   Source File: SelectorFunctionKeysTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOriginalTypes2() throws Exception {
	final TupleTypeInfo<Tuple2<Integer, String>> t1 = new TupleTypeInfo<>(
			BasicTypeInfo.INT_TYPE_INFO,
			BasicTypeInfo.STRING_TYPE_INFO
	);
	TypeInformation<PojoWithMultiplePojos> t2 = TypeExtractor.getForClass(PojoWithMultiplePojos.class);

	Keys<PojoWithMultiplePojos> sk = new Keys.SelectorFunctionKeys<>(
			new KeySelector3(),
			t2,
			t1
	);

	Assert.assertArrayEquals(
			new TypeInformation<?>[] { t1 },
			sk.getOriginalKeyFieldTypes()
	);
}
 
Example 10
Source Project: flink   Source File: AvroInputFormatTypeExtractionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTypeExtraction() {
	try {
		InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);

		TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);

		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		DataSet<MyAvroType> input = env.createInput(format);
		TypeInformation<?> typeInfoDataSet = input.getType();

		Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
		Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);

		Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
		Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 11
Source Project: flink   Source File: BroadcastConnectedStream.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Assumes as inputs a {@link BroadcastStream} and a non-keyed {@link DataStream} and applies the given
 * {@link BroadcastProcessFunction} on them, thereby creating a transformed output stream.
 *
 * @param function The {@link BroadcastProcessFunction} that is called for each element in the stream.
 * @param <OUT> The type of the output elements.
 * @return The transformed {@link DataStream}.
 */
@PublicEvolving
public <OUT> SingleOutputStreamOperator<OUT> process(final BroadcastProcessFunction<IN1, IN2, OUT> function) {

	TypeInformation<OUT> outTypeInfo = TypeExtractor.getBinaryOperatorReturnType(
			function,
			BroadcastProcessFunction.class,
			0,
			1,
			2,
			TypeExtractor.NO_INDEX,
			getType1(),
			getType2(),
			Utils.getCallLocationName(),
			true);

	return process(function, outTypeInfo);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: StreamExecutionEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new data stream that contains the given elements. The elements must all be of the
 * same type, for example, all of the {@link String} or {@link Integer}.
 *
 * <p>The framework will try and determine the exact type from the elements. In case of generic
 * elements, it may be necessary to manually supply the type information via
 * {@link #fromCollection(java.util.Collection, org.apache.flink.api.common.typeinfo.TypeInformation)}.
 *
 * <p>Note that this operation will result in a non-parallel data stream source, i.e. a data
 * stream source with a degree of parallelism one.
 *
 * @param data
 * 		The array of elements to create the data stream from.
 * @param <OUT>
 * 		The type of the returned data stream
 * @return The data stream representing the given array of elements
 */
@SafeVarargs
public final <OUT> DataStreamSource<OUT> fromElements(OUT... data) {
	if (data.length == 0) {
		throw new IllegalArgumentException("fromElements needs at least one element as argument");
	}

	TypeInformation<OUT> typeInfo;
	try {
		typeInfo = TypeExtractor.getForObject(data[0]);
	}
	catch (Exception e) {
		throw new RuntimeException("Could not create TypeInformation for type " + data[0].getClass().getName()
				+ "; please specify the TypeInformation manually via "
				+ "StreamExecutionEnvironment#fromElements(Collection, TypeInformation)", e);
	}
	return fromCollection(Arrays.asList(data), typeInfo);
}
 
Example 13
Source Project: flink   Source File: WritableSerializerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringArrayWritable() {
	StringArrayWritable[] data = new StringArrayWritable[]{
			new StringArrayWritable(new String[]{}),
			new StringArrayWritable(new String[]{""}),
			new StringArrayWritable(new String[]{"a", "a"}),
			new StringArrayWritable(new String[]{"a", "b"}),
			new StringArrayWritable(new String[]{"c", "c"}),
			new StringArrayWritable(new String[]{"d", "f"}),
			new StringArrayWritable(new String[]{"d", "m"}),
			new StringArrayWritable(new String[]{"z", "x"}),
			new StringArrayWritable(new String[]{"a", "a", "a"})
	};

	WritableTypeInfo<StringArrayWritable> writableTypeInfo = (WritableTypeInfo<StringArrayWritable>) TypeExtractor.getForObject(data[0]);
	WritableSerializer<StringArrayWritable> writableSerializer = (WritableSerializer<StringArrayWritable>) writableTypeInfo.createSerializer(new ExecutionConfig());

	SerializerTestInstance<StringArrayWritable> testInstance = new SerializerTestInstance<StringArrayWritable>(writableSerializer, writableTypeInfo.getTypeClass(), -1, data);

	testInstance.testAll();
}
 
Example 14
Source Project: Flink-CEPplus   Source File: AllGroupReduceDriverTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllReduceDriverImmutableEmpty() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TypeInformation<Tuple2<String, Integer>> typeInfo = TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		context.setDriverStrategy(DriverStrategy.ALL_GROUP_REDUCE);
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(new DiscardingOutputCollector<Tuple2<String, Integer>>());
		
		AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new AllGroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 15
Source Project: Flink-CEPplus   Source File: CsvInputFormatTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPojoTypeWithMappingInformation() throws Exception {
	File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
	tempFile.deleteOnExit();
	tempFile.setWritable(true);

	OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
	wrt.write("123,3.123,AAA,BBB\n");
	wrt.write("456,1.123,BBB,AAA\n");
	wrt.close();

	@SuppressWarnings("unchecked")
	PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
	CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[]{"field1", "field3", "field2", "field4"});

	inputFormat.configure(new Configuration());
	FileInputSplit[] splits = inputFormat.createInputSplits(1);

	inputFormat.open(splits[0]);

	validatePojoItem(inputFormat);
}
 
Example 16
Source Project: flink   Source File: UserDefinedFunctionHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tries to infer the TypeInformation of an AggregateFunction's accumulator type.
 *
 * @param aggregateFunction The AggregateFunction for which the accumulator type is inferred.
 * @param scalaType The implicitly inferred type of the accumulator type.
 *
 * @return The inferred accumulator type of the AggregateFunction.
 */
public static <T, ACC> TypeInformation<ACC> getAccumulatorTypeOfAggregateFunction(
		UserDefinedAggregateFunction<T, ACC> aggregateFunction,
		TypeInformation<ACC> scalaType) {

	TypeInformation<ACC> userProvidedType = aggregateFunction.getAccumulatorType();
	if (userProvidedType != null) {
		return userProvidedType;
	} else if (scalaType != null) {
		return scalaType;
	} else {
		return TypeExtractor.createTypeInfo(
			aggregateFunction,
			UserDefinedAggregateFunction.class,
			aggregateFunction.getClass(),
			1);
	}
}
 
Example 17
Source Project: flink   Source File: JoinedStreams.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Completes the join operation with the user function that is executed
 * for each combination of elements with the same key in a window.
 *
 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 * Due to binary backwards compatibility, this cannot be altered. Use the
 * {@link #with(FlatJoinFunction)}, method to set an operator-specific parallelism.
 */
public <T> DataStream<T> apply(FlatJoinFunction<T1, T2, T> function) {
	TypeInformation<T> resultType = TypeExtractor.getBinaryOperatorReturnType(
		function,
		FlatJoinFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		input1.getType(),
		input2.getType(),
		"Join",
		false);

	return apply(function, resultType);
}
 
Example 18
Source Project: Flink-CEPplus   Source File: Keys.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public <E> void validateCustomPartitioner(Partitioner<E> partitioner, TypeInformation<E> typeInfo) {

	if (keyFields.size() != 1) {
		throw new InvalidProgramException("Custom partitioners can only be used with keys that have one key field.");
	}
	
	if (typeInfo == null) {
		// try to extract key type from partitioner
		try {
			typeInfo = TypeExtractor.getPartitionerTypes(partitioner);
		}
		catch (Throwable t) {
			// best effort check, so we ignore exceptions
		}
	}

	// only check if type is known and not a generic type
	if (typeInfo != null && !(typeInfo instanceof GenericTypeInfo)) {
		// check equality of key and partitioner type
		if (!keyType.equals(typeInfo)) {
			throw new InvalidProgramException("The partitioner is incompatible with the key type. "
				+ "Partitioner type: " + typeInfo + " , key type: " + keyType);
		}
	}
}
 
Example 19
Source Project: flink   Source File: AvroRecordInputFormatTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test validates proper serialization with specific (generated POJO) types.
 */
@Test
public void testDeserializeToSpecificType() throws IOException {

	DatumReader<User> datumReader = new SpecificDatumReader<>(userSchema);

	try (FileReader<User> dataFileReader = DataFileReader.openReader(testFile, datumReader)) {
		User rec = dataFileReader.next();

		// check if record has been read correctly
		assertNotNull(rec);
		assertEquals("name not equal", TEST_NAME, rec.get("name").toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), rec.get("type_enum").toString());

		// now serialize it with our framework:
		ExecutionConfig ec = new ExecutionConfig();
		TypeInformation<User> te = TypeExtractor.createTypeInfo(User.class);

		assertEquals(AvroTypeInfo.class, te.getClass());
		TypeSerializer<User> tser = te.createSerializer(ec);

		ByteArrayOutputStream out = new ByteArrayOutputStream();
		try (DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(out)) {
			tser.serialize(rec, outView);
		}

		User newRec;
		try (DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(
				new ByteArrayInputStream(out.toByteArray()))) {
			newRec = tser.deserialize(inView);
		}

		// check if it is still the same
		assertNotNull(newRec);
		assertEquals("name not equal", TEST_NAME, newRec.getName().toString());
		assertEquals("enum not equal", TEST_ENUM_COLOR.toString(), newRec.getTypeEnum().toString());
	}
}
 
Example 20
Source Project: flink   Source File: ReduceCombineDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testImmutableEmpty() {
	try {
		TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
		context.getTaskConfig().setRelativeMemoryDriver(0.5);
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		Collections.shuffle(data);
		
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		
		context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		
		ReduceCombineDriver<Tuple2<String, Integer>> driver = new ReduceCombineDriver<Tuple2<String,Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Assert.assertEquals(0, result.getList().size());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 21
Source Project: flink   Source File: OneInputOperatorTransformation.java    License: Apache License 2.0 5 votes vote down vote up
private KeyedOperatorTransformation<Tuple, T> keyBy(Keys<T> keys) {
	KeySelector<T, Tuple> keySelector = KeySelectorUtil.getSelectorForKeys(
		keys,
		dataSet.getType(),
		dataSet.getExecutionEnvironment().getConfig());

	TypeInformation<Tuple> keyType = TypeExtractor.getKeySelectorTypes(keySelector, dataSet.getType());
	return new KeyedOperatorTransformation<>(dataSet, operatorMaxParallelism, keySelector, keyType);
}
 
Example 22
Source Project: flink-simple-tutorial   Source File: InputFormatExample.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> DataSet<T> getSource(ExecutionEnvironment env, String path, String[] fieldOrder, Class<T> type) throws URISyntaxException {

        // 本地文件路径
        URL fileUrl = InputFormatExample.class.getClassLoader().getResource(path);
        Path filePath = Path.fromLocalFile(new File(fileUrl.getPath()));
        // 抽取  TypeInformation,是一个 PojoTypeInfo
        PojoTypeInfo<T> pojoType = (PojoTypeInfo<T>) TypeExtractor.createTypeInfo(type);
        // 由于 Java 反射抽取出的字段顺序是不确定的,需要显式指定下文件中字段的顺序
        // 创建 PojoCsvInputFormat
        PojoCsvInputFormat<T> csvInput = new PojoCsvInputFormat<>(filePath, pojoType, fieldOrder);
        return env.createInput(csvInput, pojoType);
    }
 
Example 23
Source Project: flink   Source File: PojoSerializer.java    License: Apache License 2.0 5 votes vote down vote up
private TypeSerializer<?> createSubclassSerializer(Class<?> subclass) {
	TypeSerializer<?> serializer = TypeExtractor.createTypeInfo(subclass).createSerializer(executionConfig);

	if (serializer instanceof PojoSerializer) {
		PojoSerializer<?> subclassSerializer = (PojoSerializer<?>) serializer;
		subclassSerializer.copyBaseFieldOrder(this);
	}

	return serializer;
}
 
Example 24
Source Project: flink   Source File: TableFunction.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns {@link TypeInformation} about the operands of the evaluation method with a given
 * signature.
 *
 * <p>In order to perform operand type inference in SQL (especially when NULL is used) it might be
 * necessary to determine the parameter {@link TypeInformation} of an evaluation method.
 * By default Flink's type extraction facilities are used for this but might be wrong for
 * more complex, custom, or composite types.
 *
 * @param signature signature of the method the operand types need to be determined
 * @return {@link TypeInformation} of operand types
 */
public TypeInformation<?>[] getParameterTypes(Class<?>[] signature) {
	final TypeInformation<?>[] types = new TypeInformation<?>[signature.length];
	for (int i = 0; i < signature.length; i++) {
		try {
			types[i] = TypeExtractor.getForClass(signature[i]);
		} catch (InvalidTypesException e) {
			throw new ValidationException(
				"Parameter types of table function " + this.getClass().getCanonicalName() +
				" cannot be automatically determined. Please provide type information manually.");
		}
	}
	return types;
}
 
Example 25
@Override
public TypeSerializer<StaticSchemaPojo> createUpgradedSerializer() {
	ExecutionConfig executionConfig = new ExecutionConfig();
	// different registration order than setup
	executionConfig.registerPojoType(StaticSchemaPojoSubclassB.class);
	executionConfig.registerPojoType(StaticSchemaPojoSubclassA.class);

	TypeSerializer<StaticSchemaPojo> serializer = TypeExtractor.createTypeInfo(StaticSchemaPojo.class).createSerializer(executionConfig);
	assertSame(serializer.getClass(), PojoSerializer.class);
	return serializer;
}
 
Example 26
Source Project: flink   Source File: FieldAccessorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test(expected = CompositeType.InvalidFieldReferenceException.class)
public void testIllegalTupleInPojoInTuple() {
	Tuple2<String, Foo> t = Tuple2.of("aa", new Foo(8, Tuple2.of("ddd", 9L), (short) 2));
	TupleTypeInfo<Tuple2<String, Foo>> tpeInfo =
		(TupleTypeInfo<Tuple2<String, Foo>>) TypeExtractor.getForObject(t);

	FieldAccessorFactory.getAccessor(tpeInfo, "illegal.illegal.illegal", null);
}
 
Example 27
Source Project: flink   Source File: ExecutionEnvironment.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Registers the given type with the serialization stack. If the type is eventually
 * serialized as a POJO, then the type is registered with the POJO serializer. If the
 * type ends up being serialized with Kryo, then it will be registered at Kryo to make
 * sure that only tags are written.
 *
 * @param type The class of the type to register.
 */
public void registerType(Class<?> type) {
	if (type == null) {
		throw new NullPointerException("Cannot register null type class.");
	}

	TypeInformation<?> typeInfo = TypeExtractor.createTypeInfo(type);

	if (typeInfo instanceof PojoTypeInfo) {
		config.registerPojoType(type);
	} else {
		config.registerKryoType(type);
	}
}
 
Example 28
Source Project: flink   Source File: ReduceCombineDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testImmutableEmpty() {
	try {
		TestTaskContext<ReduceFunction<Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<ReduceFunction<Tuple2<String,Integer>>, Tuple2<String,Integer>>(1024 * 1024);
		context.getTaskConfig().setRelativeMemoryDriver(0.5);
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		Collections.shuffle(data);
		
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = EmptyMutableObjectIterator.get();
		
		context.setDriverStrategy(DriverStrategy.SORTED_PARTIAL_REDUCE);
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setComparator1(comparator);
		context.setCollector(result);
		
		ReduceCombineDriver<Tuple2<String, Integer>> driver = new ReduceCombineDriver<Tuple2<String,Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Assert.assertEquals(0, result.getList().size());
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}
 
Example 29
Source Project: flink   Source File: HadoopReduceCombineFunction.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void open(Configuration parameters) throws Exception {
	super.open(parameters);
	this.reducer.configure(jobConf);
	this.combiner.configure(jobConf);

	this.reporter = new HadoopDummyReporter();
	Class<KEYIN> inKeyClass = (Class<KEYIN>) TypeExtractor.getParameterType(Reducer.class, reducer.getClass(), 0);
	TypeSerializer<KEYIN> keySerializer = TypeExtractor.getForClass(inKeyClass).createSerializer(getRuntimeContext().getExecutionConfig());
	this.valueIterator = new HadoopTupleUnwrappingIterator<>(keySerializer);
	this.combineCollector = new HadoopOutputCollector<>();
	this.reduceCollector = new HadoopOutputCollector<>();
}
 
Example 30
Source Project: flink   Source File: GroupReduceDriverTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllReduceDriverImmutable() {
	try {
		TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String, Integer>> context =
				new TestTaskContext<GroupReduceFunction<Tuple2<String, Integer>, Tuple2<String, Integer>>, Tuple2<String,Integer>>();
		
		List<Tuple2<String, Integer>> data = DriverTestData.createReduceImmutableData();
		TupleTypeInfo<Tuple2<String, Integer>> typeInfo = (TupleTypeInfo<Tuple2<String, Integer>>) TypeExtractor.getForObject(data.get(0));
		MutableObjectIterator<Tuple2<String, Integer>> input = new RegularToMutableObjectIterator<Tuple2<String, Integer>>(data.iterator(), typeInfo.createSerializer(new ExecutionConfig()));
		TypeComparator<Tuple2<String, Integer>> comparator = typeInfo.createComparator(new int[]{0}, new boolean[] {true}, 0, new ExecutionConfig());
		
		GatheringCollector<Tuple2<String, Integer>> result = new GatheringCollector<Tuple2<String,Integer>>(typeInfo.createSerializer(new ExecutionConfig()));
		
		context.setDriverStrategy(DriverStrategy.SORTED_GROUP_REDUCE);
		context.setInput1(input, typeInfo.createSerializer(new ExecutionConfig()));
		context.setCollector(result);
		context.setComparator1(comparator);
		context.setUdf(new ConcatSumReducer());
		
		GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>> driver = new GroupReduceDriver<Tuple2<String, Integer>, Tuple2<String, Integer>>();
		driver.setup(context);
		driver.prepare();
		driver.run();
		
		Object[] res = result.getList().toArray();
		Object[] expected = DriverTestData.createReduceImmutableDataGroupedResult().toArray();
		
		DriverTestData.compareTupleArrays(expected, res);
	}
	catch (Exception e) {
		System.err.println(e.getMessage());
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}