org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner Java Examples

The following examples show how to use org.apache.flink.streaming.runtime.partitioner.GlobalPartitioner. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: DataStreamTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example #2

Source File: StreamGraphGeneratorTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}

Example #3

Source File: DataStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example #4

Source File: StreamGraphGeneratorTest.java From flink with Apache License 2.0

4 votes

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}

Example #5

Source File: DataStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}

Example #6

Source File: StreamGraphGeneratorTest.java From flink with Apache License 2.0

4 votes

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}

Example #7

Source File: DataStream.java From Flink-CEPplus with Apache License 2.0

2 votes

/**
 * Sets the partitioning of the {@link DataStream} so that the output values
 * all go to the first instance of the next processing operator. Use this
 * setting with care since it might cause a serious performance bottleneck
 * in the application.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> global() {
	return setConnectionType(new GlobalPartitioner<T>());
}

Example #8

Source File: DataStream.java From flink with Apache License 2.0

2 votes

/**
 * Sets the partitioning of the {@link DataStream} so that the output values
 * all go to the first instance of the next processing operator. Use this
 * setting with care since it might cause a serious performance bottleneck
 * in the application.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> global() {
	return setConnectionType(new GlobalPartitioner<T>());
}

Example #9

Source File: DataStream.java From flink with Apache License 2.0

2 votes

/**
 * Sets the partitioning of the {@link DataStream} so that the output values
 * all go to the first instance of the next processing operator. Use this
 * setting with care since it might cause a serious performance bottleneck
 * in the application.
 *
 * @return The DataStream with shuffle partitioning set.
 */
@PublicEvolving
public DataStream<T> global() {
	return setConnectionType(new GlobalPartitioner<T>());
}