org.apache.flink.streaming.api.datastream.IterativeStream.ConnectedIterativeStreams Java Examples

The following examples show how to use org.apache.flink.streaming.api.datastream.IterativeStream.ConnectedIterativeStreams. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IterateITCase.java    From Flink-CEPplus with Apache License 2.0 7 votes vote down vote up 
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
Example #2
Source File: IterateITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
Example #3
Source File: IterateITCase.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testImmutabilityWithCoiteration() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap); // for rebalance

	IterativeStream<Integer> iter1 = source.iterate();
	// Calling withFeedbackType should create a new iteration
	ConnectedIterativeStreams<Integer, String> iter2 = iter1.withFeedbackType(String.class);

	iter1.closeWith(iter1.map(noOpIntMap)).print();
	iter2.closeWith(iter2.map(noOpCoMap)).print();

	StreamGraph graph = env.getStreamGraph();

	assertEquals(2, graph.getIterationSourceSinkPairs().size());

	for (Tuple2<StreamNode, StreamNode> sourceSinkPair: graph.getIterationSourceSinkPairs()) {
		assertEquals(graph.getTargetVertex(sourceSinkPair.f0.getOutEdges().get(0)),
			graph.getSourceVertex(sourceSinkPair.f1.getInEdges().get(0)));
	}
}
Example #4
Source File: IterateITCase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoIterClosingFromOutOfLoop() throws Exception {

	// this test verifies that we cannot close an iteration with a DataStream that does not
	// have the iteration in its predecessors

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	IterativeStream<Integer> iter1 = source.iterate();
	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(iter1.map(noOpIntMap));

}
Example #5
Source File: IterateITCase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoIterClosingFromOutOfLoop() throws Exception {

	// this test verifies that we cannot close an iteration with a DataStream that does not
	// have the iteration in its predecessors

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	IterativeStream<Integer> iter1 = source.iterate();
	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(iter1.map(noOpIntMap));

}
Example #6
Source File: IterateITCase.java    From flink with Apache License 2.0 4 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoIterClosingFromOutOfLoop() throws Exception {

	// this test verifies that we cannot close an iteration with a DataStream that does not
	// have the iteration in its predecessors

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	IterativeStream<Integer> iter1 = source.iterate();
	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(iter1.map(noOpIntMap));

}
Example #7
Source File: IterateITCase.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoDifferingParallelism() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(coIter.map(noOpIntCoMap).setParallelism(parallelism / 2));

}
Example #8
Source File: IterateITCase.java    From flink with Apache License 2.0 3 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoDifferingParallelism() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(coIter.map(noOpIntCoMap).setParallelism(parallelism / 2));

}
Example #9
Source File: IterateITCase.java    From flink with Apache License 2.0 3 votes vote down vote up 
@Test(expected = UnsupportedOperationException.class)
public void testCoDifferingParallelism() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	// introduce dummy mapper to get to correct parallelism
	DataStream<Integer> source = env.fromElements(1, 10).map(noOpIntMap);

	ConnectedIterativeStreams<Integer, Integer> coIter = source.iterate().withFeedbackType(
			Integer.class);

	coIter.closeWith(coIter.map(noOpIntCoMap).setParallelism(parallelism / 2));

}