Java Code Examples for org.apache.flink.api.common.functions.CoGroupFunction

The following examples show how to use org.apache.flink.api.common.functions.CoGroupFunction. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupRawOperator.java    License: Apache License 2.0 6 votes vote down vote up
public CoGroupRawOperator(DataSet<I1> input1, DataSet<I2> input2,
		Keys<I1> keys1, Keys<I2> keys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> returnType,
		String defaultName) {
	super(input1, input2, returnType);
	this.function = function;
	this.defaultName = defaultName;
	this.name = defaultName;

	if (keys1 == null || keys2 == null) {
		throw new NullPointerException();
	}

	this.keys1 = keys1;
	this.keys2 = keys2;

	extractSemanticAnnotationsFromUdf(function.getClass());
}
 
Example #2
Source Project: Flink-CEPplus   Author: ljygz   File: PlanLeftUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanLeftUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		int[] key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<I2> typeInfo2) {

	super(
			new TupleLeftUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, I2, OUT>(
					typeInfoWithKey1,
					typeInfo2,
					resultType),
			key1.computeLogicalKeyPositions(),
			key2,
			name);
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: PlanRightUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanRightUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		int[] key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<I1> typeInfo1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleRightUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<I1, Tuple2<K, I2>, OUT>(
					typeInfo1,
					typeInfoWithKey2,
					resultType),
			key1,
			key2.computeLogicalKeyPositions(),
			name);
}
 
Example #4
Source Project: Flink-CEPplus   Author: ljygz   File: PlanBothUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanBothUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> type,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleBothUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, Tuple2<K, I2>, OUT>(
					typeInfoWithKey1,
					typeInfoWithKey2,
					type),
			key1.computeLogicalKeyPositions(),
			key2.computeLogicalKeyPositions(),
			name);
}
 
Example #5
Source Project: Flink-CEPplus   Author: ljygz   File: TypeExtractor.java    License: Apache License 2.0 6 votes vote down vote up
@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(CoGroupFunction<IN1, IN2, OUT> coGroupInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) coGroupInterface,
		CoGroupFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 
Example #6
Source Project: flink   Author: apache   File: PlanBothUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanBothUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> type,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleBothUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, Tuple2<K, I2>, OUT>(
					typeInfoWithKey1,
					typeInfoWithKey2,
					type),
			key1.computeLogicalKeyPositions(),
			key2.computeLogicalKeyPositions(),
			name);
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: CoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}
 
Example #8
Source Project: flink   Author: apache   File: CoGroupRawOperator.java    License: Apache License 2.0 6 votes vote down vote up
public CoGroupRawOperator(DataSet<I1> input1, DataSet<I2> input2,
		Keys<I1> keys1, Keys<I2> keys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> returnType,
		String defaultName) {
	super(input1, input2, returnType);
	this.function = function;
	this.defaultName = defaultName;
	this.name = defaultName;

	if (keys1 == null || keys2 == null) {
		throw new NullPointerException();
	}

	this.keys1 = keys1;
	this.keys2 = keys2;

	extractSemanticAnnotationsFromUdf(function.getClass());
}
 
Example #9
Source Project: flink   Author: flink-tpc-ds   File: CoGroupRawOperator.java    License: Apache License 2.0 6 votes vote down vote up
public CoGroupRawOperator(DataSet<I1> input1, DataSet<I2> input2,
		Keys<I1> keys1, Keys<I2> keys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> returnType,
		String defaultName) {
	super(input1, input2, returnType);
	this.function = function;
	this.defaultName = defaultName;
	this.name = defaultName;

	if (keys1 == null || keys2 == null) {
		throw new NullPointerException();
	}

	this.keys1 = keys1;
	this.keys2 = keys2;

	extractSemanticAnnotationsFromUdf(function.getClass());
}
 
Example #10
Source Project: flink   Author: flink-tpc-ds   File: PlanLeftUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanLeftUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		int[] key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<I2> typeInfo2) {

	super(
			new TupleLeftUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, I2, OUT>(
					typeInfoWithKey1,
					typeInfo2,
					resultType),
			key1.computeLogicalKeyPositions(),
			key2,
			name);
}
 
Example #11
Source Project: flink   Author: flink-tpc-ds   File: PlanRightUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanRightUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		int[] key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<I1> typeInfo1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleRightUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<I1, Tuple2<K, I2>, OUT>(
					typeInfo1,
					typeInfoWithKey2,
					resultType),
			key1,
			key2.computeLogicalKeyPositions(),
			name);
}
 
Example #12
Source Project: flink   Author: flink-tpc-ds   File: PlanBothUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanBothUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		Keys.SelectorFunctionKeys<I2, K> key2,
		String name,
		TypeInformation<OUT> type,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<Tuple2<K, I2>> typeInfoWithKey2) {

	super(
			new TupleBothUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, Tuple2<K, I2>, OUT>(
					typeInfoWithKey1,
					typeInfoWithKey2,
					type),
			key1.computeLogicalKeyPositions(),
			key2.computeLogicalKeyPositions(),
			name);
}
 
Example #13
Source Project: flink   Author: apache   File: TypeExtractor.java    License: Apache License 2.0 6 votes vote down vote up
@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(CoGroupFunction<IN1, IN2, OUT> coGroupInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) coGroupInterface,
		CoGroupFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 
Example #14
Source Project: flink   Author: flink-tpc-ds   File: TypeExtractor.java    License: Apache License 2.0 6 votes vote down vote up
@PublicEvolving
public static <IN1, IN2, OUT> TypeInformation<OUT> getCoGroupReturnTypes(CoGroupFunction<IN1, IN2, OUT> coGroupInterface,
		TypeInformation<IN1> in1Type, TypeInformation<IN2> in2Type, String functionName, boolean allowMissing)
{
	return getBinaryOperatorReturnType(
		(Function) coGroupInterface,
		CoGroupFunction.class,
		0,
		1,
		2,
		new int[]{2, 0},
		in1Type,
		in2Type,
		functionName,
		allowMissing);
}
 
Example #15
Source Project: flink   Author: apache   File: PlanLeftUnwrappingCoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
public PlanLeftUnwrappingCoGroupOperator(
		CoGroupFunction<I1, I2, OUT> udf,
		Keys.SelectorFunctionKeys<I1, K> key1,
		int[] key2,
		String name,
		TypeInformation<OUT> resultType,
		TypeInformation<Tuple2<K, I1>> typeInfoWithKey1,
		TypeInformation<I2> typeInfo2) {

	super(
			new TupleLeftUnwrappingCoGrouper<I1, I2, OUT, K>(udf),
			new BinaryOperatorInformation<Tuple2<K, I1>, I2, OUT>(
					typeInfoWithKey1,
					typeInfo2,
					resultType),
			key1.computeLogicalKeyPositions(),
			key2,
			name);
}
 
Example #16
Source Project: flink   Author: apache   File: CoGroupOperator.java    License: Apache License 2.0 6 votes vote down vote up
private static <I1, I2, K, OUT> PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroup(
		SelectorFunctionKeys<I1, ?> rawKeys1, SelectorFunctionKeys<I2, ?> rawKeys2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I2, K> keys2 = (SelectorFunctionKeys<I2, K>) rawKeys2;

	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final TypeInformation<Tuple2<K, I2>> typeInfoWithKey2 = KeyFunctions.createTypeWithKey(keys2);

	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);
	final Operator<Tuple2<K, I2>> keyedInput2 = KeyFunctions.appendKeyExtractor(input2, keys2);

	final PlanBothUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
		new PlanBothUnwrappingCoGroupOperator<>(function, keys1, keys2, name, outputType, typeInfoWithKey1, typeInfoWithKey2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(keyedInput2);

	return cogroup;
}
 
Example #17
Source Project: flink-perf   Author: project-flink   File: CompareJob.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(final String[] args) throws Exception {
	// set up the execution environment
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	DataSet<AvroLineitem> lineItemFromAvro = env.createInput(
			new AvroInputFormat<AvroLineitem>(new Path(args[0]), AvroLineitem.class));
	DataSet<AvroLineitem> lineItemFromCsv = env.readTextFile(args[1]).map(new Prepare.AvroLineItemMapper());
	DataSet<String> empty = lineItemFromAvro
		.coGroup(lineItemFromCsv).where("orderKey", "partKey", "supplierKey", "lineNumber").equalTo("orderKey", "partKey", "supplierKey", "lineNumber").with(new CoGroupFunction<AvroLineitem, AvroLineitem, String>() {
			@Override
			public void coGroup(Iterable<AvroLineitem> avro, Iterable<AvroLineitem> csv, Collector<String> collector) throws Exception {
				Iterator<AvroLineitem> aIt = avro.iterator();
				if(!aIt.hasNext()) {
					throw new RuntimeException("Expected item from Avro input");
				}
				AvroLineitem left = aIt.next();
				if(aIt.hasNext()) {
					throw new RuntimeException("Unexpectedly received two avro records on this side. left="+left+" next="+aIt.next());
				}

				Iterator<AvroLineitem> cIt = csv.iterator();
				if(!cIt.hasNext()) {
					throw new RuntimeException("Expected item from CSV input");
				}
				AvroLineitem right = cIt.next();
				if(cIt.hasNext()) {
					throw new RuntimeException("Unexpectedly received two CSV records on this side");
				}
				if(!right.equals(left)) {
					throw new RuntimeException("Records are not equal");
				}
			}
		});
	empty.output(new DiscardingOutputFormat<String>());
	env.execute("Compare Job");
}
 
Example #18
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupOperator.java    License: Apache License 2.0 5 votes vote down vote up
private static <I1, I2, K, OUT> PlanLeftUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroupLeft(
		SelectorFunctionKeys<I1, ?> rawKeys1, int[] logicalKeyPositions2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<I2> inputType2, TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	if (!inputType2.isTupleType()) {
		throw new InvalidParameterException("Should not happen.");
	}

	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);

	final PlanLeftUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
			new PlanLeftUnwrappingCoGroupOperator<>(
					function,
					keys1,
					logicalKeyPositions2,
					name,
					outputType,
					typeInfoWithKey1,
					inputType2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(input2);

	return cogroup;
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: UdfAnalyzerExamplesTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWebLogAnalysisExamplesAntiJoinVisits() {
	compareAnalyzerResultWithAnnotationsDualInputWithKeys(CoGroupFunction.class, AntiJoinVisits.class,
			TypeInformation.of(new TypeHint<Tuple3<Integer, String, Integer>>(){}),
			TypeInformation.of(new TypeHint<Tuple1<String>>(){}),
			TypeInformation.of(new TypeHint<Tuple3<Integer, String, Integer>>(){}),
			new String[] { "1" }, new String[] { "0" });
}
 
Example #20
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupOperatorCollectionTest.java    License: Apache License 2.0 5 votes vote down vote up
private CoGroupOperatorBase<Tuple2<String, Integer>, Tuple2<String, Integer>,
		Tuple2<String, Integer>, CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>,
		Tuple2<String, Integer>>> getCoGroupOperator(
		RichCoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>> udf) {

	TypeInformation<Tuple2<String, Integer>> tuple2Info = TypeInformation.of(new TypeHint<Tuple2<String, Integer>>(){});

	return new CoGroupOperatorBase<>(
			udf,
			new BinaryOperatorInformation<>(tuple2Info, tuple2Info, tuple2Info),
			new int[]{0},
			new int[]{0},
			"coGroup on Collections"
	);
}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() throws Exception
{
	final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();

	final CoGroupFunction<IT1, IT2, OT> coGroupStub = this.taskContext.getStub();
	final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
	final CoGroupTaskIterator<IT1, IT2> coGroupIterator = this.coGroupIterator;
	
	while (this.running && coGroupIterator.next()) {
		coGroupStub.coGroup(coGroupIterator.getValues1(), coGroupIterator.getValues2(), collector);
	}
}
 
Example #22
Source Project: flink   Author: apache   File: CoGroupedStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	dataStream1 = env.fromElements("a1", "a2", "a3");
	dataStream2 = env.fromElements("a1", "a2");
	keySelector = element -> element;
	tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1L));
	coGroupFunction = (CoGroupFunction<String, String, String>) (first, second, out) -> out.collect("");
}
 
Example #23
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupRawDriver.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() throws Exception {
	final CoGroupFunction<IT1, IT2, OT> coGroupStub = this.taskContext.getStub();
	final Collector<OT> collector = this.taskContext.getOutputCollector();
	final SimpleIterable<IT1> i1 = this.coGroupIterator1;
	final SimpleIterable<IT2> i2 = this.coGroupIterator2;

	coGroupStub.coGroup(i1, i2, collector);
}
 
Example #24
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupedStreams.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Completes the co-group operation with the user function that is executed
 * for windowed groups.
 *
 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 * Due to binary backwards compatibility, this cannot be altered. Use the {@link #with(CoGroupFunction)}
 * method to set an operator-specific parallelism.
 */
public <T> DataStream<T> apply(CoGroupFunction<T1, T2, T> function) {

	TypeInformation<T> resultType = TypeExtractor.getCoGroupReturnTypes(
		function,
		input1.getType(),
		input2.getType(),
		"CoGroup",
		false);

	return apply(function, resultType);
}
 
Example #25
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupedStreams.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Completes the co-group operation with the user function that is executed
 * for windowed groups.
 *
 * <p>Note: This method's return type does not support setting an operator-specific parallelism.
 * Due to binary backwards compatibility, this cannot be altered. Use the
 * {@link #with(CoGroupFunction, TypeInformation)} method to set an operator-specific parallelism.
 */
public <T> DataStream<T> apply(CoGroupFunction<T1, T2, T> function, TypeInformation<T> resultType) {
	//clean the closure
	function = input1.getExecutionEnvironment().clean(function);

	UnionTypeInfo<T1, T2> unionType = new UnionTypeInfo<>(input1.getType(), input2.getType());
	UnionKeySelector<T1, T2, KEY> unionKeySelector = new UnionKeySelector<>(keySelector1, keySelector2);

	DataStream<TaggedUnion<T1, T2>> taggedInput1 = input1
			.map(new Input1Tagger<T1, T2>())
			.setParallelism(input1.getParallelism())
			.returns(unionType);
	DataStream<TaggedUnion<T1, T2>> taggedInput2 = input2
			.map(new Input2Tagger<T1, T2>())
			.setParallelism(input2.getParallelism())
			.returns(unionType);

	DataStream<TaggedUnion<T1, T2>> unionStream = taggedInput1.union(taggedInput2);

	// we explicitly create the keyed stream to manually pass the key type information in
	windowedStream =
			new KeyedStream<TaggedUnion<T1, T2>, KEY>(unionStream, unionKeySelector, keyType)
			.window(windowAssigner);

	if (trigger != null) {
		windowedStream.trigger(trigger);
	}
	if (evictor != null) {
		windowedStream.evictor(evictor);
	}
	if (allowedLateness != null) {
		windowedStream.allowedLateness(allowedLateness);
	}

	return windowedStream.apply(new CoGroupWindowFunction<T1, T2, T, KEY, W>(function), resultType);
}
 
Example #26
Source Project: Flink-CEPplus   Author: ljygz   File: CoGroupedStreamsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	dataStream1 = env.fromElements("a1", "a2", "a3");
	dataStream2 = env.fromElements("a1", "a2");
	keySelector = element -> element;
	tsAssigner = TumblingEventTimeWindows.of(Time.milliseconds(1L));
	coGroupFunction = (CoGroupFunction<String, String, String>) (first, second, out) -> out.collect("");
}
 
Example #27
Source Project: flink   Author: apache   File: CoGroupOperatorCollectionTest.java    License: Apache License 2.0 5 votes vote down vote up
private CoGroupOperatorBase<Tuple2<String, Integer>, Tuple2<String, Integer>,
		Tuple2<String, Integer>, CoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>,
		Tuple2<String, Integer>>> getCoGroupOperator(
		RichCoGroupFunction<Tuple2<String, Integer>, Tuple2<String, Integer>, Tuple2<String, Integer>> udf) {

	TypeInformation<Tuple2<String, Integer>> tuple2Info = TypeInformation.of(new TypeHint<Tuple2<String, Integer>>(){});

	return new CoGroupOperatorBase<>(
			udf,
			new BinaryOperatorInformation<>(tuple2Info, tuple2Info, tuple2Info),
			new int[]{0},
			new int[]{0},
			"coGroup on Collections"
	);
}
 
Example #28
Source Project: flink   Author: apache   File: LambdaExtractionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCoGroupLambda() {
	CoGroupFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, Double>, Tuple2<Tuple1<Integer>, String>> f = (i1, i2, o) -> {};

	TypeInformation<?> ti = TypeExtractor.getCoGroupReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, NESTED_TUPLE_DOUBLE_TYPE, null, true);
	if (!(ti instanceof MissingTypeInfo)) {
		assertTrue(ti.isTupleType());
		assertEquals(2, ti.getArity());
		assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType());
		assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO);
	}
}
 
Example #29
Source Project: flink   Author: flink-tpc-ds   File: CoGroupOperator.java    License: Apache License 2.0 5 votes vote down vote up
private static <I1, I2, K, OUT> PlanLeftUnwrappingCoGroupOperator<I1, I2, OUT, K> translateSelectorFunctionCoGroupLeft(
		SelectorFunctionKeys<I1, ?> rawKeys1, int[] logicalKeyPositions2,
		CoGroupFunction<I1, I2, OUT> function,
		TypeInformation<I2> inputType2, TypeInformation<OUT> outputType, String name,
		Operator<I1> input1, Operator<I2> input2) {
	if (!inputType2.isTupleType()) {
		throw new InvalidParameterException("Should not happen.");
	}

	@SuppressWarnings("unchecked")
	final SelectorFunctionKeys<I1, K> keys1 = (SelectorFunctionKeys<I1, K>) rawKeys1;
	final TypeInformation<Tuple2<K, I1>> typeInfoWithKey1 = KeyFunctions.createTypeWithKey(keys1);
	final Operator<Tuple2<K, I1>> keyedInput1 = KeyFunctions.appendKeyExtractor(input1, keys1);

	final PlanLeftUnwrappingCoGroupOperator<I1, I2, OUT, K> cogroup =
			new PlanLeftUnwrappingCoGroupOperator<>(
					function,
					keys1,
					logicalKeyPositions2,
					name,
					outputType,
					typeInfoWithKey1,
					inputType2);

	cogroup.setFirstInput(keyedInput1);
	cogroup.setSecondInput(input2);

	return cogroup;
}
 
Example #30
Source Project: flink   Author: flink-tpc-ds   File: LambdaExtractionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCoGroupLambda() {
	CoGroupFunction<Tuple2<Tuple1<Integer>, Boolean>, Tuple2<Tuple1<Integer>, Double>, Tuple2<Tuple1<Integer>, String>> f = (i1, i2, o) -> {};

	TypeInformation<?> ti = TypeExtractor.getCoGroupReturnTypes(f, NESTED_TUPLE_BOOLEAN_TYPE, NESTED_TUPLE_DOUBLE_TYPE, null, true);
	if (!(ti instanceof MissingTypeInfo)) {
		assertTrue(ti.isTupleType());
		assertEquals(2, ti.getArity());
		assertTrue(((TupleTypeInfo<?>) ti).getTypeAt(0).isTupleType());
		assertEquals(((TupleTypeInfo<?>) ti).getTypeAt(1), BasicTypeInfo.STRING_TYPE_INFO);
	}
}