org.apache.flink.streaming.runtime.partitioner.BroadcastPartitioner Java Examples

public void finish() {

		List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
		outEdgesInOrder.add(
			new StreamEdge(
				new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null, null),
				new StreamNode(chainIndex , null, null, (StreamOperator<?>) null, null, null, null),
				0,
				Collections.<String>emptyList(),
				new BroadcastPartitioner<Object>(),
				null));

		tailConfig.setBufferTimeout(0);
		tailConfig.setChainEnd();
		tailConfig.setOutputSelectors(Collections.emptyList());
		tailConfig.setNumberOfOutputs(1);
		tailConfig.setOutEdgesInOrder(outEdgesInOrder);
		tailConfig.setNonChainedOutputs(outEdgesInOrder);
		headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
		headConfig.setOutEdgesInOrder(outEdgesInOrder);
	}
 

public MockStreamConfig(Configuration configuration, int numberOfOutputs) {
	super(configuration);

	setChainStart();
	setOutputSelectors(Collections.emptyList());
	setNumberOfOutputs(numberOfOutputs);
	setTypeSerializerOut(new StringSerializer());
	setVertexID(0);
	setStreamOperator(new TestSequentialReadingStreamOperator("test operator"));
	setOperatorID(new OperatorID());

	StreamOperator dummyOperator = new AbstractStreamOperator() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertex = new StreamNode(0, null, null, dummyOperator, "source", new ArrayList<>(), SourceStreamTask.class);
	StreamNode targetVertex = new StreamNode(1, null, null, dummyOperator, "target", new ArrayList<>(), SourceStreamTask.class);

	List<StreamEdge> outEdgesInOrder = new ArrayList<>(numberOfOutputs);
	for (int i = 0; i < numberOfOutputs; i++) {
		outEdgesInOrder.add(
			new StreamEdge(sourceVertex, targetVertex, numberOfOutputs, new ArrayList<>(), new BroadcastPartitioner<>(), null));
	}
	setOutEdgesInOrder(outEdgesInOrder);
	setNonChainedOutputs(outEdgesInOrder);
}
 

public OWNER finish() {
	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	outEdgesInOrder.add(
		new StreamEdge(
			new StreamNode(chainIndex, null, null, (StreamOperator<?>) null, null, null, null),
			new StreamNode(chainIndex , null, null, (StreamOperator<?>) null, null, null, null),
			0,
			Collections.<String>emptyList(),
			new BroadcastPartitioner<Object>(),
			null));

	tailConfig.setChainEnd();
	tailConfig.setOutputSelectors(Collections.emptyList());
	tailConfig.setNumberOfOutputs(1);
	tailConfig.setOutEdgesInOrder(outEdgesInOrder);
	tailConfig.setNonChainedOutputs(outEdgesInOrder);
	headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
	headConfig.setOutEdgesInOrder(outEdgesInOrder);

	return owner;
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<>(), new BroadcastPartitioner<>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

@Override
public void invoke() throws Exception {
	ResultPartitionWriter resultPartitionWriter = getEnvironment().getWriter(0);
	RecordWriterBuilder<LongValue> recordWriterBuilder = new RecordWriterBuilder<>();
	if (getEnvironment().getExecutionConfig().getExecutionMode() == ExecutionMode.PIPELINED) {
		// enable output flush for pipeline mode
		recordWriterBuilder.setTimeout(100);
	}
	if (useBroadcastPartitioner) {
		recordWriterBuilder.setChannelSelector(new BroadcastPartitioner());
	}
	RecordWriter<LongValue> writer = recordWriterBuilder.build(resultPartitionWriter);

	for (int i = 0; i < NUM_RECORDS_TO_SEND; ++i) {
		writer.broadcastEmit(RECORD_TO_SEND);
	}
	writer.flushAll();
	writer.clearBuffers();
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public void setupOutputForSingletonOperatorChain() {
	Preconditions.checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setBufferTimeout(0);
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);
	streamConfig.setOperatorID(new OperatorID(4711L, 123L));

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(null, 0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(null, 1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);
}
 

public void finish() {

		List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
		outEdgesInOrder.add(
			new StreamEdge(
				new StreamNode(null, chainIndex, null, null, null, null, null, null),
				new StreamNode(null, chainIndex , null, null, null, null, null, null),
				0,
				Collections.<String>emptyList(),
				new BroadcastPartitioner<Object>(),
				null));

		tailConfig.setBufferTimeout(0);
		tailConfig.setChainEnd();
		tailConfig.setOutputSelectors(Collections.emptyList());
		tailConfig.setNumberOfOutputs(1);
		tailConfig.setOutEdgesInOrder(outEdgesInOrder);
		tailConfig.setNonChainedOutputs(outEdgesInOrder);
		headConfig.setTransitiveChainedTaskConfigs(chainedConfigs);
		headConfig.setOutEdgesInOrder(outEdgesInOrder);
	}
 

/**
 * Users of the test harness can call this utility method to setup the stream config
 * if there will only be a single operator to be tested. The method will setup the
 * outgoing network connection for the operator.
 *
 * <p>For more advanced test cases such as testing chains of multiple operators with the harness,
 * please manually configure the stream config.
 */
public StreamTaskMailboxTestHarnessBuilder<OUT> setupOutputForSingletonOperatorChain(
		StreamOperatorFactory<?> factory,
		OperatorID operatorID) {
	checkState(!setupCalled, "This harness was already setup.");
	setupCalled = true;
	streamConfig.setChainStart();
	streamConfig.setTimeCharacteristic(TimeCharacteristic.EventTime);
	streamConfig.setOutputSelectors(Collections.<OutputSelector<?>>emptyList());
	streamConfig.setNumberOfOutputs(1);
	streamConfig.setTypeSerializerOut(outputSerializer);
	streamConfig.setVertexID(0);

	StreamOperator<OUT> dummyOperator = new AbstractStreamOperator<OUT>() {
		private static final long serialVersionUID = 1L;
	};

	List<StreamEdge> outEdgesInOrder = new LinkedList<StreamEdge>();
	StreamNode sourceVertexDummy = new StreamNode(0, "group", null, dummyOperator, "source dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "group", null, dummyOperator, "target dummy", new LinkedList<OutputSelector<?>>(), SourceStreamTask.class);

	outEdgesInOrder.add(new StreamEdge(sourceVertexDummy, targetVertexDummy, 0, new LinkedList<String>(), new BroadcastPartitioner<Object>(), null /* output tag */));

	streamConfig.setOutEdgesInOrder(outEdgesInOrder);
	streamConfig.setNonChainedOutputs(outEdgesInOrder);

	streamConfig.setStreamOperatorFactory(factory);
	streamConfig.setOperatorID(operatorID);

	return this;
}
 

@Override
protected void initializeInputs(StreamMockEnvironment streamMockEnvironment) {
	inputGates = new StreamTestSingleInputGate[inputSerializers.size()];
	List<StreamEdge> inPhysicalEdges = new LinkedList<>();

	StreamOperator<?> dummyOperator = new AbstractStreamOperator<Object>() {
		private static final long serialVersionUID = 1L;
	};

	StreamNode sourceVertexDummy = new StreamNode(0, "default group", null, dummyOperator, "source dummy", new LinkedList<>(), SourceStreamTask.class);
	StreamNode targetVertexDummy = new StreamNode(1, "default group", null, dummyOperator, "target dummy", new LinkedList<>(), SourceStreamTask.class);

	for (int i = 0; i < inputSerializers.size(); i++) {
		TypeSerializer<?> inputSerializer = inputSerializers.get(i);
		inputGates[i] = new StreamTestSingleInputGate<>(
			inputChannelsPerGate.get(i),
			i,
			inputSerializer,
			bufferSize);

		StreamEdge streamEdge = new StreamEdge(
			sourceVertexDummy,
			targetVertexDummy,
			i + 1,
			new LinkedList<>(),
			new BroadcastPartitioner<>(),
			null);

		inPhysicalEdges.add(streamEdge);
		streamMockEnvironment.addInputGate(inputGates[i].getInputGate());
	}

	streamConfig.setInPhysicalEdges(inPhysicalEdges);
	streamConfig.setNumberOfInputs(inputGates.length);
	streamConfig.setTypeSerializersIn(inputSerializers.toArray(new TypeSerializer[inputSerializers.size()]));
}
 

protected void setChannelSelector(RecordWriterBuilder recordWriterBuilder, boolean broadcastMode) {
	if (broadcastMode) {
		recordWriterBuilder.setChannelSelector(new BroadcastPartitioner());
	}
}
 

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			env.getStreamGraph().getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

@Test
public void testChannelSelectors() {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<Long> src = env.generateSequence(0, 0);

	DataStream<Long> broadcast = src.broadcast();
	DataStreamSink<Long> broadcastSink = broadcast.print();
	StreamPartitioner<?> broadcastPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					broadcastSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(broadcastPartitioner instanceof BroadcastPartitioner);

	DataStream<Long> shuffle = src.shuffle();
	DataStreamSink<Long> shuffleSink = shuffle.print();
	StreamPartitioner<?> shufflePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					shuffleSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(shufflePartitioner instanceof ShufflePartitioner);

	DataStream<Long> forward = src.forward();
	DataStreamSink<Long> forwardSink = forward.print();
	StreamPartitioner<?> forwardPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					forwardSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(forwardPartitioner instanceof ForwardPartitioner);

	DataStream<Long> rebalance = src.rebalance();
	DataStreamSink<Long> rebalanceSink = rebalance.print();
	StreamPartitioner<?> rebalancePartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					rebalanceSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(rebalancePartitioner instanceof RebalancePartitioner);

	DataStream<Long> global = src.global();
	DataStreamSink<Long> globalSink = global.print();
	StreamPartitioner<?> globalPartitioner =
			getStreamGraph(env).getStreamEdges(src.getId(),
					globalSink.getTransformation().getId()).get(0).getPartitioner();
	assertTrue(globalPartitioner instanceof GlobalPartitioner);
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Checks whether output selector, partitioning works correctly when applied on a union.
 */
@Test
public void testVirtualTransformations2() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	DataStream<Integer> map1 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = rebalanceMap
			.map(new NoOpIntMap());

	EvenOddOutputSelector selector = new EvenOddOutputSelector();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.broadcast()
			.split(selector)
			.select("foo")
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// verify that the properties are correctly set on all input operators
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map1.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map2.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map2.getId()).getOutputSelectors().contains(selector));

	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map3.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("foo"));
	assertTrue(graph.getStreamNode(map3.getId()).getOutputSelectors().contains(selector));

}
 

/**
 * This tests whether virtual Transformations behave correctly.
 *
 * <p>Verifies that partitioning, output selector, selected names are correctly set in the
 * StreamGraph when they are intermixed.
 */
@Test
public void testVirtualTransformations() throws Exception {

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStream<Integer> source = env.fromElements(1, 10);

	DataStream<Integer> rebalanceMap = source.rebalance().map(new NoOpIntMap());

	// verify that only the partitioning that was set last is used
	DataStream<Integer> broadcastMap = rebalanceMap
			.forward()
			.global()
			.broadcast()
			.map(new NoOpIntMap());

	broadcastMap.addSink(new DiscardingSink<>());

	// verify that partitioning is preserved across union and split/select
	EvenOddOutputSelector selector1 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector2 = new EvenOddOutputSelector();
	EvenOddOutputSelector selector3 = new EvenOddOutputSelector();

	DataStream<Integer> map1Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map1 = map1Operator
			.broadcast()
			.split(selector1)
			.select("even");

	DataStream<Integer> map2Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map2 = map2Operator
			.split(selector2)
			.select("odd")
			.global();

	DataStream<Integer> map3Operator = rebalanceMap
			.map(new NoOpIntMap());

	DataStream<Integer> map3 = map3Operator
			.global()
			.split(selector3)
			.select("even")
			.shuffle();

	SingleOutputStreamOperator<Integer> unionedMap = map1.union(map2).union(map3)
			.map(new NoOpIntMap());

	unionedMap.addSink(new DiscardingSink<>());

	StreamGraph graph = env.getStreamGraph();

	// rebalanceMap
	assertTrue(graph.getStreamNode(rebalanceMap.getId()).getInEdges().get(0).getPartitioner() instanceof RebalancePartitioner);

	// verify that only last partitioning takes precedence
	assertTrue(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertEquals(rebalanceMap.getId(), graph.getSourceVertex(graph.getStreamNode(broadcastMap.getId()).getInEdges().get(0)).getId());

	// verify that partitioning in unions is preserved and that it works across split/select
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof BroadcastPartitioner);
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map1Operator.getId()).getOutputSelectors().contains(selector1));

	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof GlobalPartitioner);
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("odd"));
	assertTrue(graph.getStreamNode(map2Operator.getId()).getOutputSelectors().contains(selector2));

	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getPartitioner() instanceof ShufflePartitioner);
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutEdges().get(0).getSelectedNames().get(0).equals("even"));
	assertTrue(graph.getStreamNode(map3Operator.getId()).getOutputSelectors().contains(selector3));
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcasted to every parallel instance of the next operation. In addition,
 * it implicitly as many {@link org.apache.flink.api.common.state.BroadcastState broadcast states}
 * as the specified descriptors which can be used to store the element of the stream.
 *
 * @param broadcastStateDescriptors the descriptors of the broadcast states to create.
 * @return A {@link BroadcastStream} which can be used in the {@link #connect(BroadcastStream)} to
 * create a {@link BroadcastConnectedStream} for further processing of the elements.
 */
@PublicEvolving
public BroadcastStream<T> broadcast(final MapStateDescriptor<?, ?>... broadcastStateDescriptors) {
	Preconditions.checkNotNull(broadcastStateDescriptors);
	final DataStream<T> broadcastStream = setConnectionType(new BroadcastPartitioner<>());
	return new BroadcastStream<>(environment, broadcastStream, broadcastStateDescriptors);
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcasted to every parallel instance of the next operation. In addition,
 * it implicitly as many {@link org.apache.flink.api.common.state.BroadcastState broadcast states}
 * as the specified descriptors which can be used to store the element of the stream.
 *
 * @param broadcastStateDescriptors the descriptors of the broadcast states to create.
 * @return A {@link BroadcastStream} which can be used in the {@link #connect(BroadcastStream)} to
 * create a {@link BroadcastConnectedStream} for further processing of the elements.
 */
@PublicEvolving
public BroadcastStream<T> broadcast(final MapStateDescriptor<?, ?>... broadcastStateDescriptors) {
	Preconditions.checkNotNull(broadcastStateDescriptors);
	final DataStream<T> broadcastStream = setConnectionType(new BroadcastPartitioner<>());
	return new BroadcastStream<>(environment, broadcastStream, broadcastStateDescriptors);
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcasted to every parallel instance of the next operation. In addition,
 * it implicitly as many {@link org.apache.flink.api.common.state.BroadcastState broadcast states}
 * as the specified descriptors which can be used to store the element of the stream.
 *
 * @param broadcastStateDescriptors the descriptors of the broadcast states to create.
 * @return A {@link BroadcastStream} which can be used in the {@link #connect(BroadcastStream)} to
 * create a {@link BroadcastConnectedStream} for further processing of the elements.
 */
@PublicEvolving
public BroadcastStream<T> broadcast(final MapStateDescriptor<?, ?>... broadcastStateDescriptors) {
	Preconditions.checkNotNull(broadcastStateDescriptors);
	final DataStream<T> broadcastStream = setConnectionType(new BroadcastPartitioner<>());
	return new BroadcastStream<>(environment, broadcastStream, broadcastStateDescriptors);
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcasted to every parallel instance of the next operation.
 *
 * @return The DataStream with broadcast partitioning set.
 */
public DataStream<T> broadcast() {
	return setConnectionType(new BroadcastPartitioner<T>());
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcast to every parallel instance of the next operation.
 *
 * @return The DataStream with broadcast partitioning set.
 */
public DataStream<T> broadcast() {
	return setConnectionType(new BroadcastPartitioner<T>());
}
 

/**
 * Sets the partitioning of the {@link DataStream} so that the output elements
 * are broadcasted to every parallel instance of the next operation.
 *
 * @return The DataStream with broadcast partitioning set.
 */
public DataStream<T> broadcast() {
	return setConnectionType(new BroadcastPartitioner<T>());
}