Java Code Examples for org.apache.flink.optimizer.plan.Channel#isOnDynamicPath()

The following examples show how to use org.apache.flink.optimizer.plan.Channel#isOnDynamicPath() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: HashJoinBuildFirstProperties.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in1.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		in1.setTempMode(in1.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 2

Source File: HashJoinBuildSecondProperties.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in2.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		
		in2.setTempMode(in2.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 3

Source File: HashJoinBuildFirstProperties.java From flink with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in1.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		in1.setTempMode(in1.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 4

Source File: HashJoinBuildSecondProperties.java From flink with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in2.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		
		in2.setTempMode(in2.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 5

Source File: HashJoinBuildFirstProperties.java From flink with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in1.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		in1.setTempMode(in1.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 6

Source File: HashJoinBuildSecondProperties.java From flink with Apache License 2.0

6 votes

@Override
public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) {
	DriverStrategy strategy;
	
	if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) {
		// sanity check that the first input is cached and remove that cache
		if (!in2.getTempMode().isCached()) {
			throw new CompilerException("No cache at point where static and dynamic parts meet.");
		}
		
		in2.setTempMode(in2.getTempMode().makeNonCached());
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED;
	}
	else {
		strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND;
	}
	return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2);
}

Example 7

Source File: JobGraphGenerator.java From Flink-CEPplus with Apache License 2.0

4 votes

private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
	// serializer
	if (isBroadcastChannel) {
		config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
		
		if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
			throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
		} else {
			return;
		}
	} else {
		config.setInputSerializer(channel.getSerializer(), inputNum);
	}
	
	// local strategy
	if (channel.getLocalStrategy() != LocalStrategy.NONE) {
		config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
		if (channel.getLocalStrategyComparator() != null) {
			config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
		}
	}
	
	assignLocalStrategyResources(channel, config, inputNum);
	
	// materialization / caching
	if (channel.getTempMode() != null) {
		final TempMode tm = channel.getTempMode();

		boolean needsMemory = false;
		// Don't add a pipeline breaker if the data exchange is already blocking, EXCEPT the channel is within an iteration.
		if (tm.breaksPipeline() &&
				(channel.isOnDynamicPath() || channel.getDataExchangeMode() != DataExchangeMode.BATCH) ) {
			config.setInputAsynchronouslyMaterialized(inputNum, true);
			needsMemory = true;
		}
		if (tm.isCached()) {
			config.setInputCached(inputNum, true);
			needsMemory = true;
		}
		
		if (needsMemory) {
			// sanity check
			if (tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
				throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
			}
			config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
		}
	}
}

Example 8

Source File: SingleInputNode.java From Flink-CEPplus with Apache License 2.0

4 votes

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
		List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
{
	final PlanNode inputSource = in.getSource();
	
	for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
		
		boolean validCombination = true;
		boolean requiresPipelinebreaker = false;
		
		// check whether the broadcast inputs use the same plan candidate at the branching point
		for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
			NamedChannel nc = broadcastChannelsCombination.get(i);
			PlanNode bcSource = nc.getSource();
			
			// check branch compatibility against input
			if (!areBranchCompatible(bcSource, inputSource)) {
				validCombination = false;
				break;
			}
			
			// check branch compatibility against all other broadcast variables
			for (int k = 0; k < i; k++) {
				PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
				
				if (!areBranchCompatible(bcSource, otherBcSource)) {
					validCombination = false;
					break;
				}
			}
			
			// check if there is a common predecessor and whether there is a dam on the way to all common predecessors
			if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
				for (OptimizerNode brancher : this.hereJoinedBranches) {
					PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
					
					if (candAtBrancher == null) {
						// closed branch between two broadcast variables
						continue;
					}
					
					SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
					if (res == NOT_FOUND) {
						throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
					} else if (res == FOUND_SOURCE) {
						requiresPipelinebreaker = true;
						break;
					} else if (res == FOUND_SOURCE_AND_DAM) {
						// good
					} else {
						throw new CompilerException();
					}
				}
			}
		}
		
		if (!validCombination) {
			continue;
		}
		
		if (requiresPipelinebreaker) {
			in.setTempMode(in.getTempMode().makePipelineBreaker());
		}
		
		final SingleInputPlanNode node = dps.instantiate(in, this);
		node.setBroadcastInputs(broadcastChannelsCombination);
		
		// compute how the strategy affects the properties
		GlobalProperties gProps = in.getGlobalProperties().clone();
		LocalProperties lProps = in.getLocalProperties().clone();
		gProps = dps.computeGlobalProperties(gProps);
		lProps = dps.computeLocalProperties(lProps);

		// filter by the user code field copies
		gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
		lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
		
		// apply
		node.initProperties(gProps, lProps);
		node.updatePropertiesWithUniqueSets(getUniqueFields());
		target.add(node);
	}
}

Example 9

Source File: JobGraphGenerator.java From flink with Apache License 2.0

4 votes

private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
	// serializer
	if (isBroadcastChannel) {
		config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
		
		if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
			throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
		} else {
			return;
		}
	} else {
		config.setInputSerializer(channel.getSerializer(), inputNum);
	}
	
	// local strategy
	if (channel.getLocalStrategy() != LocalStrategy.NONE) {
		config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
		if (channel.getLocalStrategyComparator() != null) {
			config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
		}
	}
	
	assignLocalStrategyResources(channel, config, inputNum);
	
	// materialization / caching
	if (channel.getTempMode() != null) {
		final TempMode tm = channel.getTempMode();

		boolean needsMemory = false;
		// Don't add a pipeline breaker if the data exchange is already blocking, EXCEPT the channel is within an iteration.
		if (tm.breaksPipeline() &&
				(channel.isOnDynamicPath() || channel.getDataExchangeMode() != DataExchangeMode.BATCH) ) {
			config.setInputAsynchronouslyMaterialized(inputNum, true);
			needsMemory = true;
		}
		if (tm.isCached()) {
			config.setInputCached(inputNum, true);
			needsMemory = true;
		}
		
		if (needsMemory) {
			// sanity check
			if (tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
				throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
			}
			config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
		}
	}
}

Example 10

Source File: SingleInputNode.java From flink with Apache License 2.0

4 votes

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
		List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
{
	final PlanNode inputSource = in.getSource();
	
	for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
		
		boolean validCombination = true;
		boolean requiresPipelinebreaker = false;
		
		// check whether the broadcast inputs use the same plan candidate at the branching point
		for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
			NamedChannel nc = broadcastChannelsCombination.get(i);
			PlanNode bcSource = nc.getSource();
			
			// check branch compatibility against input
			if (!areBranchCompatible(bcSource, inputSource)) {
				validCombination = false;
				break;
			}
			
			// check branch compatibility against all other broadcast variables
			for (int k = 0; k < i; k++) {
				PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
				
				if (!areBranchCompatible(bcSource, otherBcSource)) {
					validCombination = false;
					break;
				}
			}
			
			// check if there is a common predecessor and whether there is a dam on the way to all common predecessors
			if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
				for (OptimizerNode brancher : this.hereJoinedBranches) {
					PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
					
					if (candAtBrancher == null) {
						// closed branch between two broadcast variables
						continue;
					}
					
					SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
					if (res == NOT_FOUND) {
						throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
					} else if (res == FOUND_SOURCE) {
						requiresPipelinebreaker = true;
						break;
					} else if (res == FOUND_SOURCE_AND_DAM) {
						// good
					} else {
						throw new CompilerException();
					}
				}
			}
		}
		
		if (!validCombination) {
			continue;
		}
		
		if (requiresPipelinebreaker) {
			in.setTempMode(in.getTempMode().makePipelineBreaker());
		}
		
		final SingleInputPlanNode node = dps.instantiate(in, this);
		node.setBroadcastInputs(broadcastChannelsCombination);
		
		// compute how the strategy affects the properties
		GlobalProperties gProps = in.getGlobalProperties().clone();
		LocalProperties lProps = in.getLocalProperties().clone();
		gProps = dps.computeGlobalProperties(gProps);
		lProps = dps.computeLocalProperties(lProps);

		// filter by the user code field copies
		gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
		lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
		
		// apply
		node.initProperties(gProps, lProps);
		node.updatePropertiesWithUniqueSets(getUniqueFields());
		target.add(node);
	}
}

Example 11

Source File: JobGraphGenerator.java From flink with Apache License 2.0

4 votes

private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
	// serializer
	if (isBroadcastChannel) {
		config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
		
		if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
			throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
		} else {
			return;
		}
	} else {
		config.setInputSerializer(channel.getSerializer(), inputNum);
	}
	
	// local strategy
	if (channel.getLocalStrategy() != LocalStrategy.NONE) {
		config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
		if (channel.getLocalStrategyComparator() != null) {
			config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
		}
	}
	
	assignLocalStrategyResources(channel, config, inputNum);
	
	// materialization / caching
	if (channel.getTempMode() != null) {
		final TempMode tm = channel.getTempMode();

		boolean needsMemory = false;
		// Don't add a pipeline breaker if the data exchange is already blocking, EXCEPT the channel is within an iteration.
		if (tm.breaksPipeline() &&
				(channel.isOnDynamicPath() || channel.getDataExchangeMode() != DataExchangeMode.BATCH) ) {
			config.setInputAsynchronouslyMaterialized(inputNum, true);
			needsMemory = true;
		}
		if (tm.isCached()) {
			config.setInputCached(inputNum, true);
			needsMemory = true;
		}
		
		if (needsMemory) {
			// sanity check
			if (tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
				throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
			}
			config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
		}
	}
}

Example 12

Source File: SingleInputNode.java From flink with Apache License 2.0

4 votes

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
		List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
{
	final PlanNode inputSource = in.getSource();
	
	for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
		
		boolean validCombination = true;
		boolean requiresPipelinebreaker = false;
		
		// check whether the broadcast inputs use the same plan candidate at the branching point
		for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
			NamedChannel nc = broadcastChannelsCombination.get(i);
			PlanNode bcSource = nc.getSource();
			
			// check branch compatibility against input
			if (!areBranchCompatible(bcSource, inputSource)) {
				validCombination = false;
				break;
			}
			
			// check branch compatibility against all other broadcast variables
			for (int k = 0; k < i; k++) {
				PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
				
				if (!areBranchCompatible(bcSource, otherBcSource)) {
					validCombination = false;
					break;
				}
			}
			
			// check if there is a common predecessor and whether there is a dam on the way to all common predecessors
			if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
				for (OptimizerNode brancher : this.hereJoinedBranches) {
					PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
					
					if (candAtBrancher == null) {
						// closed branch between two broadcast variables
						continue;
					}
					
					SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
					if (res == NOT_FOUND) {
						throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
					} else if (res == FOUND_SOURCE) {
						requiresPipelinebreaker = true;
						break;
					} else if (res == FOUND_SOURCE_AND_DAM) {
						// good
					} else {
						throw new CompilerException();
					}
				}
			}
		}
		
		if (!validCombination) {
			continue;
		}
		
		if (requiresPipelinebreaker) {
			in.setTempMode(in.getTempMode().makePipelineBreaker());
		}
		
		final SingleInputPlanNode node = dps.instantiate(in, this);
		node.setBroadcastInputs(broadcastChannelsCombination);
		
		// compute how the strategy affects the properties
		GlobalProperties gProps = in.getGlobalProperties().clone();
		LocalProperties lProps = in.getLocalProperties().clone();
		gProps = dps.computeGlobalProperties(gProps);
		lProps = dps.computeLocalProperties(lProps);

		// filter by the user code field copies
		gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
		lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
		
		// apply
		node.initProperties(gProps, lProps);
		node.updatePropertiesWithUniqueSets(getUniqueFields());
		target.add(node);
	}
}