Java Code Examples for org.apache.flink.runtime.operators.shipping.ShipStrategyType#FORWARD

public void collect(Channel in, List<Channel> inputs) {
	if (in.getSource() instanceof NAryUnionPlanNode) {
		// sanity check
		if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
			throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
		}
		if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
			throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
		}

		inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
	} else {
		// is not a collapsed union node, so we take the channel directly
		inputs.add(in);
	}
}
 

private boolean checkRepartitionShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.PARTITION_HASH == join.getInput1().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == join.getInput2().getShipStrategy() &&
		ShipStrategyType.FORWARD == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNull("Plan should not have a combiner", combiner);
		return true;
	} else {
		return false;
	}
}
 

private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() &&
		ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNotNull("Plan should have a combiner", combiner);
		Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
		return true;
	} else {
		return false;
	}
}
 

private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() &&
		ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNotNull("Plan should have a combiner", combiner);
		Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
		return true;
	} else {
		return false;
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_REDUCE);
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_GROUP_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());

		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder());
		return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_GROUP_REDUCE);
	}
}
 

private boolean checkBroadcastShipStrategies(DualInputPlanNode join, SingleInputPlanNode reducer,
		SingleInputPlanNode combiner) {
	if (ShipStrategyType.BROADCAST == join.getInput1().getShipStrategy() &&
		ShipStrategyType.FORWARD == join.getInput2().getShipStrategy() &&
		ShipStrategyType.PARTITION_HASH == reducer.getInput().getShipStrategy()) {

		// check combiner
		Assert.assertNotNull("Plan should have a combiner", combiner);
		Assert.assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
		return true;
	} else {
		return false;
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		// locally connected, directly instantiate
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										in, DriverStrategy.ALL_REDUCE);
	} else {
		// non forward case.plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		
		// create an input node for combine with same parallelism as input node
		ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode,
				"Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
									in.getShipStrategySortOrder(), in.getDataExchangeMode());
		toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.ALL_REDUCE);
	}
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
	// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
	final Configuration conf = getOperator().getParameters();
	ShipStrategyType preSet1 = null;
	ShipStrategyType preSet2 = null;
	
	String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
		}
	}

	// see if there is a hint that dictates which shipping strategy to use for the FIRST input
	shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet1 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
		}
	}

	// see if there is a hint that dictates which shipping strategy to use for the SECOND input
	shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet2 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
		}
	}
	
	// get the predecessors
	DualInputOperator<?, ?, ?, ?> contr = getOperator();
	
	Operator<?> leftPred = contr.getFirstInput();
	Operator<?> rightPred = contr.getSecondInput();
	
	OptimizerNode pred1;
	DagConnection conn1;
	if (leftPred == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
	} else {
		pred1 = contractToNode.get(leftPred);
		conn1 = new DagConnection(pred1, this, defaultExecutionMode);
		if (preSet1 != null) {
			conn1.setShipStrategy(preSet1);
		}
	} 
	
	// create the connection and add it
	this.input1 = conn1;
	pred1.addOutgoingConnection(conn1);
	
	OptimizerNode pred2;
	DagConnection conn2;
	if (rightPred == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
	} else {
		pred2 = contractToNode.get(rightPred);
		conn2 = new DagConnection(pred2, this, defaultExecutionMode);
		if (preSet2 != null) {
			conn2.setShipStrategy(preSet2);
		}
	}
	
	// create the connection and add it
	this.input2 = conn2;
	pred2.addOutgoingConnection(conn2);
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
		throws CompilerException
{
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	final ShipStrategyType preSet;
	
	if (shipStrategy != null) {
		if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
			preSet = ShipStrategyType.PARTITION_HASH;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
			preSet = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
			preSet = ShipStrategyType.FORWARD;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
		}
	} else {
		preSet = null;
	}
	
	// get the predecessor node
	Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
	
	OptimizerNode pred;
	DagConnection conn;
	if (children == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
	} else {
		pred = contractToNode.get(children);
		conn = new DagConnection(pred, this, defaultExchangeMode);
		if (preSet != null) {
			conn.setShipStrategy(preSet);
		}
	}
	
	// create the connection and add it
	setIncomingConnection(conn);
	pred.addOutgoingConnection(conn);
}
 

public void parameterizeChannel(Channel channel, boolean globalDopChange,
								ExecutionMode exchangeMode, boolean breakPipeline) {

	ShipStrategyType shipType;
	FieldList partitionKeys;
	boolean[] sortDirection;
	Partitioner<?> partitioner;

	switch (this.partitioning) {
		case RANDOM_PARTITIONED:
			shipType = globalDopChange ? ShipStrategyType.PARTITION_RANDOM : ShipStrategyType.FORWARD;
			partitionKeys = null;
			sortDirection = null;
			partitioner = null;
			break;

		case FULL_REPLICATION:
			shipType = ShipStrategyType.BROADCAST;
			partitionKeys = null;
			sortDirection = null;
			partitioner = null;
			break;

		case ANY_PARTITIONING:
		case HASH_PARTITIONED:
			shipType = ShipStrategyType.PARTITION_HASH;
			partitionKeys = Utils.createOrderedFromSet(this.partitioningFields);
			sortDirection = null;
			partitioner = null;
			break;

		case RANGE_PARTITIONED:
			shipType = ShipStrategyType.PARTITION_RANGE;
			partitionKeys = this.ordering.getInvolvedIndexes();
			sortDirection = this.ordering.getFieldSortDirections();
			partitioner = null;
			break;

		case FORCED_REBALANCED:
			shipType = ShipStrategyType.PARTITION_RANDOM;
			partitionKeys = null;
			sortDirection = null;
			partitioner = null;
			break;

		case CUSTOM_PARTITIONING:
			shipType = ShipStrategyType.PARTITION_CUSTOM;
			partitionKeys = this.partitioningFields;
			sortDirection = null;
			partitioner = this.customPartitioner;
			break;

		default:
			throw new CompilerException("Unsupported partitioning strategy");
	}

	channel.setDataDistribution(this.distribution);
	DataExchangeMode exMode = DataExchangeMode.select(exchangeMode, shipType, breakPipeline);
	channel.setShipStrategy(shipType, partitionKeys, sortDirection, partitioner, exMode);
}
 

public FeedbackPropertiesMeetRequirementsReport checkPartialSolutionPropertiesMet(PlanNode partialSolution, GlobalProperties feedbackGlobal, LocalProperties feedbackLocal) {
	if (this == partialSolution) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
	
	boolean found = false;
	boolean allMet = true;
	boolean allLocallyMet = true;
	
	for (Channel input : getInputs()) {
		FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
		
		if (inputState == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.MET) {
			found = true;
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
			return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
		}
		else {
			found = true;
			
			// the partial solution was on the path here. check whether the channel requires
			// certain properties that are met, or whether the channel introduces new properties
			
			// if the plan introduces new global properties, then we can stop looking whether
			// the feedback properties are sufficient to meet the requirements
			if (input.getShipStrategy() != ShipStrategyType.FORWARD && input.getShipStrategy() != ShipStrategyType.NONE) {
				continue;
			}
			
			// first check whether this channel requires something that is not met
			if (input.getRequiredGlobalProps() != null && !input.getRequiredGlobalProps().isMetBy(feedbackGlobal)) {
				return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
			}
			
			// in general, not everything is met here already
			allMet = false;
			
			// if the plan introduces new local properties, we can stop checking for matching local properties
			if (inputState != FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET) {
				
				if (input.getLocalStrategy() == LocalStrategy.NONE) {
					
					if (input.getRequiredLocalProps() != null && !input.getRequiredLocalProps().isMetBy(feedbackLocal)) {
						return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
					}
					
					allLocallyMet = false;
				}
			}
		}
	}
	
	if (!found) {
		return FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION;
	} else if (allMet) {
		return FeedbackPropertiesMeetRequirementsReport.MET;
	} else if (allLocallyMet) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET;
	} else {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
		// adjust a sort (changes grouping, so it must be for this driver to combining sort
		if (in.getLocalStrategy() == LocalStrategy.SORT) {
			if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
				throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
			}
			in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
								in.getLocalStrategySortOrder());
		}
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
										DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	} else {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator()
				.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		// set sorting comparator key info
		combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
		// set grouping comparator key info
		combiner.setDriverKeyInfo(this.keyList, 1);
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
								in.getShipStrategySortOrder(), in.getDataExchangeMode());
		if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			toReducer.setDataDistribution(in.getDataDistribution());
		}
		toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	}
}
 

/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable containingTask, ClassLoader cl, TaskConfig config,
									List<ChainedDriver<?, ?>> chainedTasksTarget,
									List<RecordWriter<?>> eventualOutputs,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?,?>> accumulatorMap)
throws Exception
{
	final int numOutputs = config.getNumOutputs();

	// check whether we got any chained tasks
	final int numChained = config.getNumberOfChainedStubs();
	if (numChained > 0) {
		// got chained stubs. that means that this one may only have a single forward connection
		if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) {
			throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection.");
		}

		// instantiate each task
		@SuppressWarnings("rawtypes")
		Collector previous = null;
		for (int i = numChained - 1; i >= 0; --i)
		{
			// get the task first
			final ChainedDriver<?, ?> ct;
			try {
				Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i);
				ct = ctc.newInstance();
			}
			catch (Exception ex) {
				throw new RuntimeException("Could not instantiate chained task driver.", ex);
			}

			// get the configuration for the task
			final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
			final String taskName = config.getChainedTaskName(i);

			if (i == numChained - 1) {
				// last in chain, instantiate the output collector for this task
				previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
			}

			ct.setup(chainedStubConf, taskName, previous, containingTask, cl, executionConfig, accumulatorMap);
			chainedTasksTarget.add(0, ct);

			if (i == numChained - 1) {
				ct.getIOMetrics().reuseOutputMetricsForTask();
			}

			previous = ct;
		}
		// the collector of the first in the chain is the collector for the task
		return (Collector<T>) previous;
	}
	// else

	// instantiate the output collector the default way from this configuration
	return getOutputCollector(containingTask , config, cl, eventualOutputs, 0, numOutputs);
}
 

public FeedbackPropertiesMeetRequirementsReport checkPartialSolutionPropertiesMet(PlanNode partialSolution, GlobalProperties feedbackGlobal, LocalProperties feedbackLocal) {
	if (this == partialSolution) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
	
	boolean found = false;
	boolean allMet = true;
	boolean allLocallyMet = true;
	
	for (Channel input : getInputs()) {
		FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
		
		if (inputState == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.MET) {
			found = true;
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
			return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
		}
		else {
			found = true;
			
			// the partial solution was on the path here. check whether the channel requires
			// certain properties that are met, or whether the channel introduces new properties
			
			// if the plan introduces new global properties, then we can stop looking whether
			// the feedback properties are sufficient to meet the requirements
			if (input.getShipStrategy() != ShipStrategyType.FORWARD && input.getShipStrategy() != ShipStrategyType.NONE) {
				continue;
			}
			
			// first check whether this channel requires something that is not met
			if (input.getRequiredGlobalProps() != null && !input.getRequiredGlobalProps().isMetBy(feedbackGlobal)) {
				return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
			}
			
			// in general, not everything is met here already
			allMet = false;
			
			// if the plan introduces new local properties, we can stop checking for matching local properties
			if (inputState != FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET) {
				
				if (input.getLocalStrategy() == LocalStrategy.NONE) {
					
					if (input.getRequiredLocalProps() != null && !input.getRequiredLocalProps().isMetBy(feedbackLocal)) {
						return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
					}
					
					allLocallyMet = false;
				}
			}
		}
	}
	
	if (!found) {
		return FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION;
	} else if (allMet) {
		return FeedbackPropertiesMeetRequirementsReport.MET;
	} else if (allLocallyMet) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET;
	} else {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExecutionMode) {
	// see if there is a hint that dictates which shipping strategy to use for BOTH inputs
	final Configuration conf = getOperator().getParameters();
	ShipStrategyType preSet1 = null;
	ShipStrategyType preSet2 = null;
	
	String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet1 = preSet2 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy: " + shipStrategy);
		}
	}

	// see if there is a hint that dictates which shipping strategy to use for the FIRST input
	shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_FIRST_INPUT, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet1 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet1 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy of input one: " + shipStrategy);
		}
	}

	// see if there is a hint that dictates which shipping strategy to use for the SECOND input
	shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY_SECOND_INPUT, null);
	if (shipStrategy != null) {
		if (Optimizer.HINT_SHIP_STRATEGY_FORWARD.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.FORWARD;
		} else if (Optimizer.HINT_SHIP_STRATEGY_BROADCAST.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.BROADCAST;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.PARTITION_HASH;
		} else if (Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE.equals(shipStrategy)) {
			preSet2 = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet2 = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unknown hint for shipping strategy of input two: " + shipStrategy);
		}
	}
	
	// get the predecessors
	DualInputOperator<?, ?, ?, ?> contr = getOperator();
	
	Operator<?> leftPred = contr.getFirstInput();
	Operator<?> rightPred = contr.getSecondInput();
	
	OptimizerNode pred1;
	DagConnection conn1;
	if (leftPred == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for first input.");
	} else {
		pred1 = contractToNode.get(leftPred);
		conn1 = new DagConnection(pred1, this, defaultExecutionMode);
		if (preSet1 != null) {
			conn1.setShipStrategy(preSet1);
		}
	} 
	
	// create the connection and add it
	this.input1 = conn1;
	pred1.addOutgoingConnection(conn1);
	
	OptimizerNode pred2;
	DagConnection conn2;
	if (rightPred == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input set for second input.");
	} else {
		pred2 = contractToNode.get(rightPred);
		conn2 = new DagConnection(pred2, this, defaultExecutionMode);
		if (preSet2 != null) {
			conn2.setShipStrategy(preSet2);
		}
	}
	
	// create the connection and add it
	this.input2 = conn2;
	pred2.addOutgoingConnection(conn2);
}
 

@Override
public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode)
		throws CompilerException
{
	// see if an internal hint dictates the strategy to use
	final Configuration conf = getOperator().getParameters();
	final String shipStrategy = conf.getString(Optimizer.HINT_SHIP_STRATEGY, null);
	final ShipStrategyType preSet;
	
	if (shipStrategy != null) {
		if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_HASH)) {
			preSet = ShipStrategyType.PARTITION_HASH;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION_RANGE)) {
			preSet = ShipStrategyType.PARTITION_RANGE;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_FORWARD)) {
			preSet = ShipStrategyType.FORWARD;
		} else if (shipStrategy.equalsIgnoreCase(Optimizer.HINT_SHIP_STRATEGY_REPARTITION)) {
			preSet = ShipStrategyType.PARTITION_RANDOM;
		} else {
			throw new CompilerException("Unrecognized ship strategy hint: " + shipStrategy);
		}
	} else {
		preSet = null;
	}
	
	// get the predecessor node
	Operator<?> children = ((SingleInputOperator<?, ?, ?>) getOperator()).getInput();
	
	OptimizerNode pred;
	DagConnection conn;
	if (children == null) {
		throw new CompilerException("Error: Node for '" + getOperator().getName() + "' has no input.");
	} else {
		pred = contractToNode.get(children);
		conn = new DagConnection(pred, this, defaultExchangeMode);
		if (preSet != null) {
			conn.setShipStrategy(preSet);
		}
	}
	
	// create the connection and add it
	setIncomingConnection(conn);
	pred.addOutgoingConnection(conn);
}
 

public FeedbackPropertiesMeetRequirementsReport checkPartialSolutionPropertiesMet(PlanNode partialSolution, GlobalProperties feedbackGlobal, LocalProperties feedbackLocal) {
	if (this == partialSolution) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
	
	boolean found = false;
	boolean allMet = true;
	boolean allLocallyMet = true;
	
	for (Channel input : getInputs()) {
		FeedbackPropertiesMeetRequirementsReport inputState = input.getSource().checkPartialSolutionPropertiesMet(partialSolution, feedbackGlobal, feedbackLocal);
		
		if (inputState == FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION) {
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.MET) {
			found = true;
			continue;
		}
		else if (inputState == FeedbackPropertiesMeetRequirementsReport.NOT_MET) {
			return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
		}
		else {
			found = true;
			
			// the partial solution was on the path here. check whether the channel requires
			// certain properties that are met, or whether the channel introduces new properties
			
			// if the plan introduces new global properties, then we can stop looking whether
			// the feedback properties are sufficient to meet the requirements
			if (input.getShipStrategy() != ShipStrategyType.FORWARD && input.getShipStrategy() != ShipStrategyType.NONE) {
				continue;
			}
			
			// first check whether this channel requires something that is not met
			if (input.getRequiredGlobalProps() != null && !input.getRequiredGlobalProps().isMetBy(feedbackGlobal)) {
				return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
			}
			
			// in general, not everything is met here already
			allMet = false;
			
			// if the plan introduces new local properties, we can stop checking for matching local properties
			if (inputState != FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET) {
				
				if (input.getLocalStrategy() == LocalStrategy.NONE) {
					
					if (input.getRequiredLocalProps() != null && !input.getRequiredLocalProps().isMetBy(feedbackLocal)) {
						return FeedbackPropertiesMeetRequirementsReport.NOT_MET;
					}
					
					allLocallyMet = false;
				}
			}
		}
	}
	
	if (!found) {
		return FeedbackPropertiesMeetRequirementsReport.NO_PARTIAL_SOLUTION;
	} else if (allMet) {
		return FeedbackPropertiesMeetRequirementsReport.MET;
	} else if (allLocallyMet) {
		return FeedbackPropertiesMeetRequirementsReport.PENDING_LOCAL_MET;
	} else {
		return FeedbackPropertiesMeetRequirementsReport.PENDING;
	}
}
 

@Override
public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) {
	if (in.getShipStrategy() == ShipStrategyType.FORWARD) {
		if(in.getSource().getOptimizerNode() instanceof PartitionNode) {
			LOG.warn("Cannot automatically inject combiner for GroupReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator.");
		}
		// adjust a sort (changes grouping, so it must be for this driver to combining sort
		if (in.getLocalStrategy() == LocalStrategy.SORT) {
			if (!in.getLocalStrategyKeys().isValidUnorderedPrefix(this.keys)) {
				throw new RuntimeException("Bug: Inconsistent sort for group strategy.");
			}
			in.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
								in.getLocalStrategySortOrder());
		}
		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in,
										DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	} else {
		// non forward case. all local properties are killed anyways, so we can safely plug in a combiner
		Channel toCombiner = new Channel(in.getSource());
		toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);

		// create an input node for combine with same parallelism as input node
		GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode();
		combinerNode.setParallelism(in.getSource().getParallelism());

		SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator()
				.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE);
		combiner.setCosts(new Costs(0, 0));
		combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
		// set sorting comparator key info
		combiner.setDriverKeyInfo(in.getLocalStrategyKeys(), in.getLocalStrategySortOrder(), 0);
		// set grouping comparator key info
		combiner.setDriverKeyInfo(this.keyList, 1);
		
		Channel toReducer = new Channel(combiner);
		toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(),
								in.getShipStrategySortOrder(), in.getDataExchangeMode());
		if (in.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
			toReducer.setDataDistribution(in.getDataDistribution());
		}
		toReducer.setLocalStrategy(LocalStrategy.COMBININGSORT, in.getLocalStrategyKeys(),
									in.getLocalStrategySortOrder());

		return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")",
										toReducer, DriverStrategy.SORTED_GROUP_REDUCE, this.keyList);
	}
}
 

/**
 * Creates a writer for each output. Creates an OutputCollector which forwards its input to all writers.
 * The output collector applies the configured shipping strategy.
 */
@SuppressWarnings("unchecked")
public static <T> Collector<T> initOutputs(AbstractInvokable containingTask, ClassLoader cl, TaskConfig config,
									List<ChainedDriver<?, ?>> chainedTasksTarget,
									List<RecordWriter<?>> eventualOutputs,
									ExecutionConfig executionConfig,
									Map<String, Accumulator<?,?>> accumulatorMap)
throws Exception
{
	final int numOutputs = config.getNumOutputs();

	// check whether we got any chained tasks
	final int numChained = config.getNumberOfChainedStubs();
	if (numChained > 0) {
		// got chained stubs. that means that this one may only have a single forward connection
		if (numOutputs != 1 || config.getOutputShipStrategy(0) != ShipStrategyType.FORWARD) {
			throw new RuntimeException("Plan Generation Bug: Found a chained stub that is not connected via an only forward connection.");
		}

		// instantiate each task
		@SuppressWarnings("rawtypes")
		Collector previous = null;
		for (int i = numChained - 1; i >= 0; --i)
		{
			// get the task first
			final ChainedDriver<?, ?> ct;
			try {
				Class<? extends ChainedDriver<?, ?>> ctc = config.getChainedTask(i);
				ct = ctc.newInstance();
			}
			catch (Exception ex) {
				throw new RuntimeException("Could not instantiate chained task driver.", ex);
			}

			// get the configuration for the task
			final TaskConfig chainedStubConf = config.getChainedStubConfig(i);
			final String taskName = config.getChainedTaskName(i);

			if (i == numChained - 1) {
				// last in chain, instantiate the output collector for this task
				previous = getOutputCollector(containingTask, chainedStubConf, cl, eventualOutputs, 0, chainedStubConf.getNumOutputs());
			}

			ct.setup(chainedStubConf, taskName, previous, containingTask, cl, executionConfig, accumulatorMap);
			chainedTasksTarget.add(0, ct);

			if (i == numChained - 1) {
				ct.getIOMetrics().reuseOutputMetricsForTask();
			}

			previous = ct;
		}
		// the collector of the first in the chain is the collector for the task
		return (Collector<T>) previous;
	}
	// else

	// instantiate the output collector the default way from this configuration
	return getOutputCollector(containingTask , config, cl, eventualOutputs, 0, numOutputs);
}