org.apache.flink.runtime.io.network.DataExchangeMode Java Examples

@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifyBranchingJoiningPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.PIPELINED,   // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.PIPELINED,   // to coGroup (first input)
			DataExchangeMode.PIPELINED,   // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH,       // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.BATCH,       // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.BATCH,       // to flatMap
			DataExchangeMode.BATCH,       // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.BATCH        // to sink after coGroup
	);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchingJoiningPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.BATCH,       // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.BATCH,       // to coGroup (first input)
			DataExchangeMode.BATCH,       // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifyBranchingJoiningPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED,   // to map
			DataExchangeMode.PIPELINED,   // to combiner connections are pipelined
			DataExchangeMode.BATCH,       // to reduce
			DataExchangeMode.BATCH,       // to filter
			DataExchangeMode.PIPELINED,   // to sink after reduce
			DataExchangeMode.PIPELINED,   // to join (first input)
			DataExchangeMode.BATCH,       // to join (second input)
			DataExchangeMode.PIPELINED,   // combiner connections are pipelined
			DataExchangeMode.BATCH,       // to other reducer
			DataExchangeMode.PIPELINED,   // to flatMap
			DataExchangeMode.PIPELINED,   // to sink after flatMap
			DataExchangeMode.PIPELINED,   // to coGroup (first input)
			DataExchangeMode.PIPELINED,   // to coGroup (second input)
			DataExchangeMode.PIPELINED    // to sink after coGroup
	);
}
 

@Test
public void testNoPartialSolutionFoundSingleInputOnly() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
		
		SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
		
		Channel toMap1 = new Channel(target);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(map1);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		{
			GlobalProperties gp = new GlobalProperties();
			LocalProperties lp = new LocalProperties();
			
			FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
			assertTrue(report == NO_PARTIAL_SOLUTION);
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

@Test
public void testPipelinedForced() {
	// PIPELINED_FORCED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_REDUCE);
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_GROUP_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());

		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_GROUP_REDUCE);
	}
}
 

@Test
public void testTwoWorksetIterationsDirectlyChained() throws Exception {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(8);
		
		DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
		
		DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
		
		DataSet<Tuple2<Long, Long>> firstResult = doDeltaIteration(verticesWithInitialId, edges);
		
		DataSet<Tuple2<Long, Long>> secondResult = doDeltaIteration(firstResult, edges);
		
		secondResult.output(new DiscardingOutputFormat<Tuple2<Long,Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		assertEquals(1, op.getDataSinks().size());
		assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
		
		WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
		
		assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy());

		assertEquals(DataExchangeMode.BATCH, wipn.getInput1().getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, wipn.getInput2().getDataExchangeMode());

		assertEquals(TempMode.NONE, wipn.getInput1().getTempMode());
		assertEquals(TempMode.NONE, wipn.getInput2().getTempMode());
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH);
}
 

@Test
public void testBreakerForDependentVariable() {
		try {
			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
			
			DataSet<String> source1 = env.fromElements("test");
			
			source1.map(new IdentityMapper<String>()).map(new IdentityMapper<String>()).withBroadcastSet(source1, "some name")
					.output(new DiscardingOutputFormat<String>());
			
			Plan p = env.createProgramPlan();
			OptimizedPlan op = compileNoStats(p);
			
			SinkPlanNode sink = op.getDataSinks().iterator().next();
			SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
			SingleInputPlanNode beforeMapper = (SingleInputPlanNode) mapper.getInput().getSource();

			assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
			assertEquals(TempMode.NONE, beforeMapper.getInput().getTempMode());
			assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode());

			assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode());
			assertEquals(DataExchangeMode.BATCH, beforeMapper.getInput().getDataExchangeMode());
			assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
		}
		catch (Exception e) {
			e.printStackTrace();
			fail(e.getMessage());
		}
}
 

@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifySimpleForwardPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED);
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_GROUP_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());

		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_GROUP_REDUCE);
	}
}
 

@Test
public void testPipelineBreakerBroadcastedAllReduce() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.getConfig().setExecutionMode(ExecutionMode.PIPELINED);
		env.setParallelism(64);
		
		DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>());
		
		DataSet<Long> bcInput1 = sourceWithMapper
									.map(new IdentityMapper<Long>())
									.reduce(new SelectOneReducer<Long>());
		DataSet<Long> bcInput2 = env.generateSequence(1, 10);
		
		DataSet<Long> result = sourceWithMapper
				.map(new IdentityMapper<Long>())
						.withBroadcastSet(bcInput1, "bc1")
						.withBroadcastSet(bcInput2, "bc2");
		
		result.output(new DiscardingOutputFormat<Long>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();

		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifyBranchigPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

@Test
public void testTwoWorksetIterationsDirectlyChained() throws Exception {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(8);
		
		DataSet<Tuple2<Long, Long>> verticesWithInitialId = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
		
		DataSet<Tuple2<Long, Long>> edges = env.fromElements(new Tuple2<Long, Long>(1L, 2L));
		
		DataSet<Tuple2<Long, Long>> firstResult = doDeltaIteration(verticesWithInitialId, edges);
		
		DataSet<Tuple2<Long, Long>> secondResult = doDeltaIteration(firstResult, edges);
		
		secondResult.output(new DiscardingOutputFormat<Tuple2<Long,Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		assertEquals(1, op.getDataSinks().size());
		assertTrue(op.getDataSinks().iterator().next().getInput().getSource() instanceof WorksetIterationPlanNode);
		
		WorksetIterationPlanNode wipn = (WorksetIterationPlanNode) op.getDataSinks().iterator().next().getInput().getSource();
		
		assertEquals(ShipStrategyType.FORWARD, wipn.getInput1().getShipStrategy());

		assertEquals(DataExchangeMode.BATCH, wipn.getInput1().getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, wipn.getInput2().getDataExchangeMode());

		assertEquals(TempMode.NONE, wipn.getInput1().getTempMode());
		assertEquals(TempMode.NONE, wipn.getInput2().getTempMode());
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testRangePartitionOperatorPreservesFields() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

		DataSet<Tuple2<Long, Long>> data = env.fromCollection(Collections.singleton(new Tuple2<>(0L, 0L)));

		data.partitionByRange(1)
			.groupBy(1)
			.reduceGroup(new IdentityGroupReducerCombinable<Tuple2<Long,Long>>())
			.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());

		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);

		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
		SingleInputPlanNode partitionNode = (SingleInputPlanNode)reducer.getInput().getSource();
		SingleInputPlanNode partitionIDRemover = (SingleInputPlanNode) partitionNode.getInput().getSource();

		assertEquals(ShipStrategyType.FORWARD, reducer.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, partitionNode.getInput().getShipStrategy());
		assertEquals(ShipStrategyType.PARTITION_CUSTOM, partitionIDRemover.getInput().getShipStrategy());

		SourcePlanNode sourcePlanNode = op.getDataSources().iterator().next();
		List<Channel> sourceOutgoingChannels = sourcePlanNode.getOutgoingChannels();
		assertEquals(2, sourceOutgoingChannels.size());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(0).getShipStrategy());
		assertEquals(ShipStrategyType.FORWARD, sourceOutgoingChannels.get(1).getShipStrategy());
		assertEquals(DataExchangeMode.PIPELINED, sourceOutgoingChannels.get(0).getDataExchangeMode());
		assertEquals(DataExchangeMode.BATCH, sourceOutgoingChannels.get(1).getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testBreakerForDependentVariable() {
		try {
			ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
			
			DataSet<String> source1 = env.fromElements("test");
			
			source1.map(new IdentityMapper<String>()).map(new IdentityMapper<String>()).withBroadcastSet(source1, "some name")
					.output(new DiscardingOutputFormat<String>());
			
			Plan p = env.createProgramPlan();
			OptimizedPlan op = compileNoStats(p);
			
			SinkPlanNode sink = op.getDataSinks().iterator().next();
			SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
			SingleInputPlanNode beforeMapper = (SingleInputPlanNode) mapper.getInput().getSource();

			assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
			assertEquals(TempMode.NONE, beforeMapper.getInput().getTempMode());
			assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode());

			assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode());
			assertEquals(DataExchangeMode.BATCH, beforeMapper.getInput().getDataExchangeMode());
			assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
		}
		catch (Exception e) {
			e.printStackTrace();
			fail(e.getMessage());
		}
}
 

@Test
public void testNoBreakerForIndependentVariable() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		
		DataSet<String> source1 = env.fromElements("test");
		DataSet<String> source2 = env.fromElements("test");
		
		source1.map(new IdentityMapper<String>()).withBroadcastSet(source2, "some name")
				.output(new DiscardingOutputFormat<String>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource();
		
		assertEquals(TempMode.NONE, mapper.getInput().getTempMode());
		assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode());
		
		assertEquals(DataExchangeMode.PIPELINED, mapper.getInput().getDataExchangeMode());
		assertEquals(DataExchangeMode.PIPELINED, mapper.getBroadcastInputs().get(0).getDataExchangeMode());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

@Test
public void testPipelined() {
	// PIPELINED should result in pipelining all the way
	verifyBranchigPlan(ExecutionMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED);
}
 

public void setShipStrategy(ShipStrategyType strategy, FieldList keys,
							boolean[] sortDirection, Partitioner<?> partitioner,
							DataExchangeMode dataExchangeMode) {
	this.shipStrategy = strategy;
	this.shipKeys = keys;
	this.shipSortOrder = sortDirection;
	this.partitioner = partitioner;
	this.dataExchangeMode = dataExchangeMode;
	this.globalProps = null;		// reset the global properties
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_REDUCE);
	}
}
 

public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
 

public UnionClosedBranchingTest(
	ExecutionMode executionMode,
	DataExchangeMode sourceToUnion,
	DataExchangeMode unionToJoin) {

	this.executionMode = executionMode;
	this.sourceToUnion = sourceToUnion;
	this.unionToJoin = unionToJoin;
}
 

@Test
public void testBatchForced() {
	// BATCH_FORCED should result in batching all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH_FORCED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.BATCH);
}
 

@Test
public void testBatch() {
	// BATCH should result in batching the shuffle all the way
	verifySimpleForwardPlan(ExecutionMode.BATCH,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED,
			DataExchangeMode.BATCH, DataExchangeMode.PIPELINED);
}