org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties Java Examples

The following examples show how to use org.apache.flink.optimizer.dataproperties.RequestedGlobalProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: PartitionNode.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	RequestedGlobalProperties rgps = new RequestedGlobalProperties();
	
	switch (this.pMethod) {
	case HASH:
		rgps.setHashPartitioned(this.keys);
		break;
	case REBALANCE:
		rgps.setForceRebalancing();
		break;
	case CUSTOM:
		rgps.setCustomPartitioned(this.keys, this.customPartitioner);
		break;
	case RANGE:
		rgps.setRangePartitioned(ordering, distribution);
		break;
	default:
		throw new IllegalArgumentException("Invalid partition method");
	}
	
	return Collections.singletonList(rgps);
}

Example #2

Source File: CartesianProductDescriptor.java From flink with Apache License 2.0

6 votes

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
	ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
	
	if (this.allowBroadcastFirst) {
		// replicate first
		RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
		replicated1.setFullyReplicated();
		RequestedGlobalProperties any2 = new RequestedGlobalProperties();
		pairs.add(new GlobalPropertiesPair(replicated1, any2));
	}
	
	if (this.allowBroadcastSecond) {
		// replicate second
		RequestedGlobalProperties any1 = new RequestedGlobalProperties();
		RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
		replicated2.setFullyReplicated();
		pairs.add(new GlobalPropertiesPair(any1, replicated2));
	}

	return pairs;
}

Example #3

Source File: DataSinkNode.java From flink with Apache License 2.0

6 votes

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties iProps = new InterestingProperties();

	{
		final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
		iProps.addGlobalProperties(partitioningProps);
	}

	{
		final Ordering localOrder = getOperator().getLocalOrder();
		final RequestedLocalProperties orderProps = new RequestedLocalProperties();
		if (localOrder != null) {
			orderProps.setOrdering(localOrder);
		}
		iProps.addLocalProperties(orderProps);
	}
	
	this.input.setInterestingProperties(iProps);
}

Example #4

Source File: DataSinkNode.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties iProps = new InterestingProperties();

	{
		final RequestedGlobalProperties partitioningProps = new RequestedGlobalProperties();
		iProps.addGlobalProperties(partitioningProps);
	}

	{
		final Ordering localOrder = getOperator().getLocalOrder();
		final RequestedLocalProperties orderProps = new RequestedLocalProperties();
		if (localOrder != null) {
			orderProps.setOrdering(localOrder);
		}
		iProps.addLocalProperties(orderProps);
	}
	
	this.input.setInterestingProperties(iProps);
}

Example #5

Source File: SingleInputNode.java From flink with Apache License 2.0

6 votes

protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps,
		List<PlanNode> target, CostEstimator estimator)
{
	for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) {
		final Channel in = template.clone();
		ilp.parameterizeChannel(in);
		
		// instantiate a candidate, if the instantiated local properties meet one possible local property set
		outer:
		for (OperatorDescriptorSingle dps: getPossibleProperties()) {
			for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) {
				if (ilps.isMetBy(in.getLocalProperties())) {
					in.setRequiredLocalProps(ilps);
					instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp);
					break outer;
				}
			}
		}
	}
}

Example #6

Source File: CartesianProductDescriptor.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
	ArrayList<GlobalPropertiesPair> pairs = new ArrayList<GlobalPropertiesPair>();
	
	if (this.allowBroadcastFirst) {
		// replicate first
		RequestedGlobalProperties replicated1 = new RequestedGlobalProperties();
		replicated1.setFullyReplicated();
		RequestedGlobalProperties any2 = new RequestedGlobalProperties();
		pairs.add(new GlobalPropertiesPair(replicated1, any2));
	}
	
	if (this.allowBroadcastSecond) {
		// replicate second
		RequestedGlobalProperties any1 = new RequestedGlobalProperties();
		RequestedGlobalProperties replicated2 = new RequestedGlobalProperties();
		replicated2.setFullyReplicated();
		pairs.add(new GlobalPropertiesPair(any1, replicated2));
	}

	return pairs;
}

Example #7

Source File: PartitionNode.java From flink with Apache License 2.0

6 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	RequestedGlobalProperties rgps = new RequestedGlobalProperties();
	
	switch (this.pMethod) {
	case HASH:
		rgps.setHashPartitioned(this.keys);
		break;
	case REBALANCE:
		rgps.setForceRebalancing();
		break;
	case CUSTOM:
		rgps.setCustomPartitioned(this.keys, this.customPartitioner);
		break;
	case RANGE:
		rgps.setRangePartitioned(ordering, distribution);
		break;
	default:
		throw new IllegalArgumentException("Invalid partition method");
	}
	
	return Collections.singletonList(rgps);
}

Example #8

Source File: CoGroupRawDescriptor.java From flink with Apache License 2.0

5 votes

@Override
protected List<OperatorDescriptorDual.GlobalPropertiesPair> createPossibleGlobalProperties() {
	RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
	partitioned1.setHashPartitioned(this.keys1);
	RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
	partitioned2.setHashPartitioned(this.keys2);
	return Collections.singletonList(new OperatorDescriptorDual.GlobalPropertiesPair(partitioned1, partitioned2));
}

Example #9

Source File: CoGroupDescriptor.java From flink with Apache License 2.0

5 votes

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {

	if (this.customPartitioner == null) {

		// we accept compatible partitionings of any type
		RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
		RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
		partitioned_left_any.setAnyPartitioning(this.keys1);
		partitioned_right_any.setAnyPartitioning(this.keys2);

		// add strict hash partitioning of both inputs on their full key sets
		RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
		RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
		partitioned_left_hash.setHashPartitioned(this.keys1);
		partitioned_right_hash.setHashPartitioned(this.keys2);

		return Arrays.asList(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any),
				new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
	}
	else {
		RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
		partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);

		RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
		partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);

		return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
	}
}

Example #10

Source File: CoGroupDescriptor.java From flink with Apache License 2.0

5 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{

	if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
			produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {

		// both are hash partitioned, check that partitioning fields are equivalently chosen
		return checkEquivalentFieldPositionsInKeyFields(
				produced1.getPartitioningFields(), produced2.getPartitioningFields());

	}
	else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
			produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
			produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {

		return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
				checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
				produced1.getDataDistribution().equals(produced2.getDataDistribution());
		
	}
	else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
			produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {

		// both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
		return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
				produced2.getPartitioningFields().isExactMatch(this.keys2) &&
				produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
				produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());

	}
	else {

		// no other partitioning valid, incl. ANY_PARTITIONING.
		//   For co-groups we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
		return false;
	}

}

Example #11

Source File: CoGroupRawDescriptor.java From flink with Apache License 2.0

5 votes

@Override
protected List<OperatorDescriptorDual.GlobalPropertiesPair> createPossibleGlobalProperties() {
	RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
	partitioned1.setHashPartitioned(this.keys1);
	RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
	partitioned2.setHashPartitioned(this.keys2);
	return Collections.singletonList(new OperatorDescriptorDual.GlobalPropertiesPair(partitioned1, partitioned2));
}

Example #12

Source File: GroupReduceProperties.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	RequestedGlobalProperties props = new RequestedGlobalProperties();
	
	if (customPartitioner == null) {
		props.setAnyPartitioning(this.keys);
	} else {
		props.setCustomPartitioned(this.keys, this.customPartitioner);
	}
	return Collections.singletonList(props);
}

Example #13

Source File: CoGroupRawDescriptor.java From flink with Apache License 2.0

5 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{
	return produced1.getPartitioning() == produced2.getPartitioning() && 
			(produced1.getCustomPartitioner() == null ? 
				produced2.getCustomPartitioner() == null :
				produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner()));
}

Example #14

Source File: CoGroupRawDescriptor.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
protected List<OperatorDescriptorDual.GlobalPropertiesPair> createPossibleGlobalProperties() {
	RequestedGlobalProperties partitioned1 = new RequestedGlobalProperties();
	partitioned1.setHashPartitioned(this.keys1);
	RequestedGlobalProperties partitioned2 = new RequestedGlobalProperties();
	partitioned2.setHashPartitioned(this.keys2);
	return Collections.singletonList(new OperatorDescriptorDual.GlobalPropertiesPair(partitioned1, partitioned2));
}

Example #15

Source File: CoGroupRawDescriptor.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{
	return produced1.getPartitioning() == produced2.getPartitioning() && 
			(produced1.getCustomPartitioner() == null ? 
				produced2.getCustomPartitioner() == null :
				produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner()));
}

Example #16

Source File: ReduceProperties.java From flink with Apache License 2.0

5 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	RequestedGlobalProperties props = new RequestedGlobalProperties();
	if (customPartitioner == null) {
		props.setAnyPartitioning(this.keys);
	} else {
		props.setCustomPartitioned(this.keys, this.customPartitioner);
	}
	return Collections.singletonList(props);
}

Example #17

Source File: GroupReduceWithCombineProperties.java From flink with Apache License 2.0

5 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	RequestedGlobalProperties props = new RequestedGlobalProperties();
	if (customPartitioner == null) {
		props.setAnyPartitioning(this.keys);
	} else {
		props.setCustomPartitioned(this.keys, this.customPartitioner);
	}
	return Collections.singletonList(props);
}

Example #18

Source File: CoGroupDescriptor.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {

	if (this.customPartitioner == null) {

		// we accept compatible partitionings of any type
		RequestedGlobalProperties partitioned_left_any = new RequestedGlobalProperties();
		RequestedGlobalProperties partitioned_right_any = new RequestedGlobalProperties();
		partitioned_left_any.setAnyPartitioning(this.keys1);
		partitioned_right_any.setAnyPartitioning(this.keys2);

		// add strict hash partitioning of both inputs on their full key sets
		RequestedGlobalProperties partitioned_left_hash = new RequestedGlobalProperties();
		RequestedGlobalProperties partitioned_right_hash = new RequestedGlobalProperties();
		partitioned_left_hash.setHashPartitioned(this.keys1);
		partitioned_right_hash.setHashPartitioned(this.keys2);

		return Arrays.asList(new GlobalPropertiesPair(partitioned_left_any, partitioned_right_any),
				new GlobalPropertiesPair(partitioned_left_hash, partitioned_right_hash));
	}
	else {
		RequestedGlobalProperties partitioned_left = new RequestedGlobalProperties();
		partitioned_left.setCustomPartitioned(this.keys1, this.customPartitioner);

		RequestedGlobalProperties partitioned_right = new RequestedGlobalProperties();
		partitioned_right.setCustomPartitioned(this.keys2, this.customPartitioner);

		return Collections.singletonList(new GlobalPropertiesPair(partitioned_left, partitioned_right));
	}
}

Example #19

Source File: SingleInputNode.java From flink with Apache License 2.0

5 votes

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	// get what we inherit and what is preserved by our user code 
	final InterestingProperties props = getInterestingProperties().filterByCodeAnnotations(this, 0);
	
	// add all properties relevant to this node
	for (OperatorDescriptorSingle dps : getPossibleProperties()) {
		for (RequestedGlobalProperties gp : dps.getPossibleGlobalProperties()) {
			
			if (gp.getPartitioning().isPartitionedOnKey()) {
				// make sure that among the same partitioning types, we do not push anything down that has fewer key fields
				
				for (RequestedGlobalProperties contained : props.getGlobalProperties()) {
					if (contained.getPartitioning() == gp.getPartitioning() && gp.getPartitionedFields().isValidSubset(contained.getPartitionedFields())) {
						props.getGlobalProperties().remove(contained);
						break;
					}
				}
			}
			
			props.addGlobalProperties(gp);
		}
		
		for (RequestedLocalProperties lp : dps.getPossibleLocalProperties()) {
			props.addLocalProperties(lp);
		}
	}
	this.inConn.setInterestingProperties(props);
	
	for (DagConnection conn : getBroadcastConnections()) {
		conn.setInterestingProperties(new InterestingProperties());
	}
}

Example #20

Source File: UtilSinkJoinOpDescriptor.java From flink with Apache License 2.0

4 votes

@Override
protected List<GlobalPropertiesPair> createPossibleGlobalProperties() {
	// all properties are possible
	return Collections.singletonList(new GlobalPropertiesPair(
		new RequestedGlobalProperties(), new RequestedGlobalProperties()));
}

Example #21

Source File: OperatorDescriptorDual.java From Flink-CEPplus with Apache License 2.0

4 votes

public abstract boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
GlobalProperties produced1, GlobalProperties produced2);

Example #22

Source File: AllGroupCombineProperties.java From flink with Apache License 2.0

4 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	return Collections.singletonList(new RequestedGlobalProperties());
}

Example #23

Source File: BulkIterationNode.java From flink with Apache License 2.0

4 votes

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties intProps = getInterestingProperties().clone();
	
	if (this.terminationCriterion != null) {
		// first propagate through termination Criterion. since it has no successors, it has no
		// interesting properties
		this.terminationCriterionRootConnection.setInterestingProperties(new InterestingProperties());
		this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator));
	}
	
	// we need to make 2 interesting property passes, because the root of the step function needs also
	// the interesting properties as generated by the partial solution
	
	// give our own interesting properties (as generated by the iterations successors) to the step function and
	// make the first pass
	this.rootConnection.setInterestingProperties(intProps);
	this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
	
	// take the interesting properties of the partial solution and add them to the root interesting properties
	InterestingProperties partialSolutionIntProps = this.partialSolution.getInterestingProperties();
	intProps.getGlobalProperties().addAll(partialSolutionIntProps.getGlobalProperties());
	intProps.getLocalProperties().addAll(partialSolutionIntProps.getLocalProperties());
	
	// clear all interesting properties to prepare the second traversal
	// this clears only the path down from the next partial solution. The paths down
	// from the termination criterion (before they meet the paths down from the next partial solution)
	// remain unaffected by this step
	this.rootConnection.clearInterestingProperties();
	this.nextPartialSolution.accept(InterestingPropertiesClearer.INSTANCE);
	
	// 2nd pass
	this.rootConnection.setInterestingProperties(intProps);
	this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
	
	// now add the interesting properties of the partial solution to the input
	final InterestingProperties inProps = this.partialSolution.getInterestingProperties().clone();
	inProps.addGlobalProperties(new RequestedGlobalProperties());
	inProps.addLocalProperties(new RequestedLocalProperties());
	this.inConn.setInterestingProperties(inProps);
}

Example #24

Source File: SingleInputNode.java From flink with Apache License 2.0

4 votes

protected void instantiateCandidate(OperatorDescriptorSingle dps, Channel in, List<Set<? extends NamedChannel>> broadcastPlanChannels,
		List<PlanNode> target, CostEstimator estimator, RequestedGlobalProperties globPropsReq, RequestedLocalProperties locPropsReq)
{
	final PlanNode inputSource = in.getSource();
	
	for (List<NamedChannel> broadcastChannelsCombination: Sets.cartesianProduct(broadcastPlanChannels)) {
		
		boolean validCombination = true;
		boolean requiresPipelinebreaker = false;
		
		// check whether the broadcast inputs use the same plan candidate at the branching point
		for (int i = 0; i < broadcastChannelsCombination.size(); i++) {
			NamedChannel nc = broadcastChannelsCombination.get(i);
			PlanNode bcSource = nc.getSource();
			
			// check branch compatibility against input
			if (!areBranchCompatible(bcSource, inputSource)) {
				validCombination = false;
				break;
			}
			
			// check branch compatibility against all other broadcast variables
			for (int k = 0; k < i; k++) {
				PlanNode otherBcSource = broadcastChannelsCombination.get(k).getSource();
				
				if (!areBranchCompatible(bcSource, otherBcSource)) {
					validCombination = false;
					break;
				}
			}
			
			// check if there is a common predecessor and whether there is a dam on the way to all common predecessors
			if (in.isOnDynamicPath() && this.hereJoinedBranches != null) {
				for (OptimizerNode brancher : this.hereJoinedBranches) {
					PlanNode candAtBrancher = in.getSource().getCandidateAtBranchPoint(brancher);
					
					if (candAtBrancher == null) {
						// closed branch between two broadcast variables
						continue;
					}
					
					SourceAndDamReport res = in.getSource().hasDamOnPathDownTo(candAtBrancher);
					if (res == NOT_FOUND) {
						throw new CompilerException("Bug: Tracing dams for deadlock detection is broken.");
					} else if (res == FOUND_SOURCE) {
						requiresPipelinebreaker = true;
						break;
					} else if (res == FOUND_SOURCE_AND_DAM) {
						// good
					} else {
						throw new CompilerException();
					}
				}
			}
		}
		
		if (!validCombination) {
			continue;
		}
		
		if (requiresPipelinebreaker) {
			in.setTempMode(in.getTempMode().makePipelineBreaker());
		}
		
		final SingleInputPlanNode node = dps.instantiate(in, this);
		node.setBroadcastInputs(broadcastChannelsCombination);
		
		// compute how the strategy affects the properties
		GlobalProperties gProps = in.getGlobalProperties().clone();
		LocalProperties lProps = in.getLocalProperties().clone();
		gProps = dps.computeGlobalProperties(gProps);
		lProps = dps.computeLocalProperties(lProps);

		// filter by the user code field copies
		gProps = gProps.filterBySemanticProperties(getSemanticPropertiesForGlobalPropertyFiltering(), 0);
		lProps = lProps.filterBySemanticProperties(getSemanticPropertiesForLocalPropertyFiltering(), 0);
		
		// apply
		node.initProperties(gProps, lProps);
		node.updatePropertiesWithUniqueSets(getUniqueFields());
		target.add(node);
	}
}

Example #25

Source File: AbstractJoinDescriptor.java From flink with Apache License 2.0

4 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{
	if (requested1.getPartitioning().isPartitionedOnKey() && requested2.getPartitioning().isPartitionedOnKey()) {

		if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
				produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {

			// both are hash partitioned, check that partitioning fields are equivalently chosen
			return checkEquivalentFieldPositionsInKeyFields(
					produced1.getPartitioningFields(), produced2.getPartitioningFields());

		}
		else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
				produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
				produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {

			return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
					checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
					produced1.getDataDistribution().equals(produced2.getDataDistribution());

		}
		else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
				produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {

			// both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
			return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
					produced2.getPartitioningFields().isExactMatch(this.keys2) &&
					produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
					produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());

		}
		else {

			// no other partitioning valid, incl. ANY_PARTITIONING.
			//   For joins we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
			return false;
		}

	} else {
		return true;
	}

}

Example #26

Source File: AllReduceProperties.java From flink with Apache License 2.0

4 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	return Collections.singletonList(new RequestedGlobalProperties());
}

Example #27

Source File: CartesianProductDescriptor.java From flink with Apache License 2.0

4 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2) {
	return true;
}

Example #28

Source File: AbstractJoinDescriptor.java From flink with Apache License 2.0

4 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{
	if (requested1.getPartitioning().isPartitionedOnKey() && requested2.getPartitioning().isPartitionedOnKey()) {

		if(produced1.getPartitioning() == PartitioningProperty.HASH_PARTITIONED &&
				produced2.getPartitioning() == PartitioningProperty.HASH_PARTITIONED) {

			// both are hash partitioned, check that partitioning fields are equivalently chosen
			return checkEquivalentFieldPositionsInKeyFields(
					produced1.getPartitioningFields(), produced2.getPartitioningFields());

		}
		else if(produced1.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
				produced2.getPartitioning() == PartitioningProperty.RANGE_PARTITIONED &&
				produced1.getDataDistribution() != null && produced2.getDataDistribution() != null) {

			return produced1.getPartitioningFields().size() == produced2.getPartitioningFields().size() &&
					checkSameOrdering(produced1, produced2, produced1.getPartitioningFields().size()) &&
					produced1.getDataDistribution().equals(produced2.getDataDistribution());

		}
		else if(produced1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING &&
				produced2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING) {

			// both use a custom partitioner. Check that both keys are exactly as specified and that both the same partitioner
			return produced1.getPartitioningFields().isExactMatch(this.keys1) &&
					produced2.getPartitioningFields().isExactMatch(this.keys2) &&
					produced1.getCustomPartitioner() != null && produced2.getCustomPartitioner() != null &&
					produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner());

		}
		else {

			// no other partitioning valid, incl. ANY_PARTITIONING.
			//   For joins we must ensure that both sides are exactly identically partitioned, ANY is not good enough.
			return false;
		}

	} else {
		return true;
	}

}

Example #29

Source File: PartialGroupProperties.java From flink with Apache License 2.0

4 votes

@Override
protected List<RequestedGlobalProperties> createPossibleGlobalProperties() {
	return Collections.singletonList(new RequestedGlobalProperties());
}

Example #30

Source File: BulkIterationNode.java From flink with Apache License 2.0

4 votes

@Override
public void computeInterestingPropertiesForInputs(CostEstimator estimator) {
	final InterestingProperties intProps = getInterestingProperties().clone();
	
	if (this.terminationCriterion != null) {
		// first propagate through termination Criterion. since it has no successors, it has no
		// interesting properties
		this.terminationCriterionRootConnection.setInterestingProperties(new InterestingProperties());
		this.terminationCriterion.accept(new InterestingPropertyVisitor(estimator));
	}
	
	// we need to make 2 interesting property passes, because the root of the step function needs also
	// the interesting properties as generated by the partial solution
	
	// give our own interesting properties (as generated by the iterations successors) to the step function and
	// make the first pass
	this.rootConnection.setInterestingProperties(intProps);
	this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
	
	// take the interesting properties of the partial solution and add them to the root interesting properties
	InterestingProperties partialSolutionIntProps = this.partialSolution.getInterestingProperties();
	intProps.getGlobalProperties().addAll(partialSolutionIntProps.getGlobalProperties());
	intProps.getLocalProperties().addAll(partialSolutionIntProps.getLocalProperties());
	
	// clear all interesting properties to prepare the second traversal
	// this clears only the path down from the next partial solution. The paths down
	// from the termination criterion (before they meet the paths down from the next partial solution)
	// remain unaffected by this step
	this.rootConnection.clearInterestingProperties();
	this.nextPartialSolution.accept(InterestingPropertiesClearer.INSTANCE);
	
	// 2nd pass
	this.rootConnection.setInterestingProperties(intProps);
	this.nextPartialSolution.accept(new InterestingPropertyVisitor(estimator));
	
	// now add the interesting properties of the partial solution to the input
	final InterestingProperties inProps = this.partialSolution.getInterestingProperties().clone();
	inProps.addGlobalProperties(new RequestedGlobalProperties());
	inProps.addLocalProperties(new RequestedLocalProperties());
	this.inConn.setInterestingProperties(inProps);
}