org.apache.flink.optimizer.dataproperties.GlobalProperties Java Examples

The following examples show how to use org.apache.flink.optimizer.dataproperties.GlobalProperties. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: BulkPartialSolutionPlanNode.java From Flink-CEPplus with Apache License 2.0

6 votes

public BulkPartialSolutionPlanNode(BulkPartialSolutionNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the partial solution does not cost anything
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}

Example #2

Source File: WorksetPlanNode.java From Flink-CEPplus with Apache License 2.0

6 votes

public WorksetPlanNode(WorksetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}

Example #3

Source File: SolutionSetPlanNode.java From Flink-CEPplus with Apache License 2.0

6 votes

public SolutionSetPlanNode(SolutionSetNode template, String nodeName,
		GlobalProperties gProps, LocalProperties lProps,
		Channel initialInput)
{
	super(template, nodeName, DriverStrategy.NONE);
	
	this.globalProps = gProps;
	this.localProps = lProps;
	this.initialInput = initialInput;
	
	// the node incurs no cost
	this.nodeCosts = NO_COSTS;
	this.cumulativeCosts = NO_COSTS;
	
	if (initialInput.getSource().branchPlan != null && initialInput.getSource().branchPlan.size() > 0) {
		if (this.branchPlan == null) {
			this.branchPlan = new HashMap<OptimizerNode, PlanNode>();
		}
		
		this.branchPlan.putAll(initialInput.getSource().branchPlan);
	}
}

Example #4

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource5() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
		.splitsPartitionedBy("f1.intField")
		.splitsOrderedBy("f0; f1.intField", new Order[]{Order.ASCENDING, Order.DESCENDING});

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(2)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0,2)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #5

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource7() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
			.splitsPartitionedBy("f1")
			.splitsOrderedBy("f1.stringField", new Order[]{Order.ASCENDING});

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #6

Source File: CoGroupDescriptor.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
	GlobalProperties gp = GlobalProperties.combine(in1, in2);
	if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
				gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gp.clearUniqueFieldCombinations();
	return gp;
}

Example #7

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedGroupedSource4() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(0, 1)
			.splitsGroupedBy(0);

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #8

Source File: CoGroupDescriptor.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties in1, GlobalProperties in2) {
	GlobalProperties gp = GlobalProperties.combine(in1, in2);
	if (gp.getUniqueFieldCombination() != null && gp.getUniqueFieldCombination().size() > 0 &&
				gp.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gp.setAnyPartitioning(gp.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gp.clearUniqueFieldCombinations();
	return gp;
}

Example #9

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource2() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(1)
			.splitsOrderedBy(new int[]{1, 0}, new Order[]{Order.ASCENDING, Order.DESCENDING});

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1, 0)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #10

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedGroupedSource8() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
			.splitsPartitionedBy("f1")
			.splitsGroupedBy("f1.stringField");

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #11

Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");

		SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
		SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
		
		Channel toMap1 = new Channel(source1);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(source2);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		Channel toJoin1 = new Channel(map1);
		Channel toJoin2 = new Channel(map2);
		
		toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin1.setLocalStrategy(LocalStrategy.NONE);
		toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin2.setLocalStrategy(LocalStrategy.NONE);
		
		DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
		
		FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
		assertEquals(NO_PARTIAL_SOLUTION, report);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #12

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedGroupedSource1() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(0)
			.splitsGroupedBy(0);

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(0)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #13

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedSource7() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 1, 0);

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(gprops.getCustomPartitioner() != null);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #14

Source File: AllGroupWithPartialPreGroupProperties.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
	if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
			gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gProps.clearUniqueFieldCombinations();
	return gProps;
}

Example #15

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedSource7() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 1, 0);

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(gprops.getCustomPartitioner() != null);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #16

Source File: WorksetNode.java From flink with Apache License 2.0

5 votes

public void setCandidateProperties(GlobalProperties gProps, LocalProperties lProps, Channel initialInput) {
	if (this.cachedPlans != null) {
		throw new IllegalStateException();
	} else {
		WorksetPlanNode wspn = new WorksetPlanNode(this, "Workset ("+this.getOperator().getName()+")", gProps, lProps, initialInput);
		this.cachedPlans = Collections.<PlanNode>singletonList(wspn);
	}
}

Example #17

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedGroupedSource4() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(0, 1)
			.splitsGroupedBy(0);

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(0, 1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #18

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedGroupedSource5() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
			.splitsPartitionedBy("f2")
			.splitsGroupedBy("f2");

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(4)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(new FieldSet(lprops.getGroupedFields().toArray()).equals(new FieldSet(4)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #19

Source File: FeedbackPropertiesMatchTest.java From flink with Apache License 2.0

5 votes

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");

		SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
		SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
		
		Channel toMap1 = new Channel(source1);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(source2);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		Channel toJoin1 = new Channel(map1);
		Channel toJoin2 = new Channel(map2);
		
		toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin1.setLocalStrategy(LocalStrategy.NONE);
		toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin2.setLocalStrategy(LocalStrategy.NONE);
		
		DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
		
		FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
		assertEquals(NO_PARTIAL_SOLUTION, report);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #20

Source File: FeedbackPropertiesMatchTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testNoPartialSolutionFoundTwoInputOperator() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Partial Solution");

		SourcePlanNode source1 = new SourcePlanNode(getSourceNode(), "Source 1");
		SourcePlanNode source2 = new SourcePlanNode(getSourceNode(), "Source 2");
		
		Channel toMap1 = new Channel(source1);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(source2);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		Channel toJoin1 = new Channel(map1);
		Channel toJoin2 = new Channel(map2);
		
		toJoin1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin1.setLocalStrategy(LocalStrategy.NONE);
		toJoin2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toJoin2.setLocalStrategy(LocalStrategy.NONE);
		
		DualInputPlanNode join = new DualInputPlanNode(getJoinNode(), "Join", toJoin1, toJoin2, DriverStrategy.HYBRIDHASH_BUILD_FIRST);
		
		FeedbackPropertiesMeetRequirementsReport report = join.checkPartialSolutionPropertiesMet(target, new GlobalProperties(), new LocalProperties());
		assertEquals(NO_PARTIAL_SOLUTION, report);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #21

Source File: FeedbackPropertiesMatchTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testNoPartialSolutionFoundSingleInputOnly() {
	try {
		SourcePlanNode target = new SourcePlanNode(getSourceNode(), "Source");
		
		SourcePlanNode otherTarget = new SourcePlanNode(getSourceNode(), "Source");
		
		Channel toMap1 = new Channel(target);
		toMap1.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap1.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map1 = new SingleInputPlanNode(getMapNode(), "Mapper 1", toMap1, DriverStrategy.MAP);
		
		Channel toMap2 = new Channel(map1);
		toMap2.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED);
		toMap2.setLocalStrategy(LocalStrategy.NONE);
		SingleInputPlanNode map2 = new SingleInputPlanNode(getMapNode(), "Mapper 2", toMap2, DriverStrategy.MAP);
		
		{
			GlobalProperties gp = new GlobalProperties();
			LocalProperties lp = new LocalProperties();
			
			FeedbackPropertiesMeetRequirementsReport report = map2.checkPartialSolutionPropertiesMet(otherTarget, gp, lp);
			assertTrue(report == NO_PARTIAL_SOLUTION);
		}
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #22

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource2() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(1)
			.splitsOrderedBy(new int[]{1, 0}, new Order[]{Order.ASCENDING, Order.DESCENDING});

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1, 0)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #23

Source File: AllGroupReduceProperties.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
	if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
			gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gProps.clearUniqueFieldCombinations();
	return gProps;
}

Example #24

Source File: AllGroupWithPartialPreGroupProperties.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
	if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
			gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gProps.clearUniqueFieldCombinations();
	return gProps;
}

Example #25

Source File: DataSourceNode.java From flink with Apache License 2.0

5 votes

/**
 * Creates a new DataSourceNode for the given contract.
 * 
 * @param pactContract
 *        The data source contract object.
 */
public DataSourceNode(GenericDataSourceBase<?, ?> pactContract) {
	super(pactContract);
	
	if (pactContract.getUserCodeWrapper().getUserCodeClass() == null) {
		throw new IllegalArgumentException("Input format has not been set.");
	}
	
	if (NonParallelInput.class.isAssignableFrom(pactContract.getUserCodeWrapper().getUserCodeClass())) {
		setParallelism(1);
		this.sequentialInput = true;
	} else {
		this.sequentialInput = false;
	}

	this.replicatedInput = ReplicatingInputFormat.class.isAssignableFrom(
													pactContract.getUserCodeWrapper().getUserCodeClass());

	this.gprops = new GlobalProperties();
	this.lprops = new LocalProperties();

	SplitDataProperties<?> splitProps = pactContract.getSplitDataProperties();

	if(replicatedInput) {
		this.gprops.setFullyReplicated();
		this.lprops = new LocalProperties();
	} else if (splitProps != null) {
		// configure data properties of data source using split properties
		setDataPropertiesFromSplitProperties(splitProps);
	}

}

Example #26

Source File: SourcePlanNode.java From flink with Apache License 2.0

5 votes

public SourcePlanNode(DataSourceNode template, String nodeName, GlobalProperties gprops, LocalProperties lprops) {
	super(template, nodeName, DriverStrategy.NONE);

	this.globalProps = gprops;
	this.localProps = lprops;
	updatePropertiesWithUniqueSets(template.getUniqueFields());
}

Example #27

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource1() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data.getSplitDataProperties()
			.splitsPartitionedBy(1)
			.splitsOrderedBy(new int[]{1}, new Order[]{Order.ASCENDING});

	data.output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue((new FieldSet(lprops.getGroupedFields().toArray())).equals(new FieldSet(1)));
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #28

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

5 votes

@Test
public void checkSinglePartitionedOrderedSource7() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple3<Long, SomePojo, String>> data = env.fromCollection(tuple3PojoData, tuple3PojoType);

	data.getSplitDataProperties()
			.splitsPartitionedBy("f1")
			.splitsOrderedBy("f1.stringField", new Order[]{Order.ASCENDING});

	data.output(new DiscardingOutputFormat<Tuple3<Long, SomePojo, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode = (SourcePlanNode) sinkNode.getPredecessor();

	GlobalProperties gprops = sourceNode.getGlobalProperties();
	LocalProperties lprops = sourceNode.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops.getPartitioningFields().toArray())).equals(new FieldSet(1,2,3)));
	Assert.assertTrue(gprops.getPartitioning() == PartitioningProperty.ANY_PARTITIONING);
	Assert.assertTrue(lprops.getGroupedFields() == null);
	Assert.assertTrue(lprops.getOrdering() == null);

}

Example #29

Source File: AllGroupCombineProperties.java From flink with Apache License 2.0

5 votes

@Override
public GlobalProperties computeGlobalProperties(GlobalProperties gProps) {
	if (gProps.getUniqueFieldCombination() != null && gProps.getUniqueFieldCombination().size() > 0 &&
			gProps.getPartitioning() == PartitioningProperty.RANDOM_PARTITIONED)
	{
		gProps.setAnyPartitioning(gProps.getUniqueFieldCombination().iterator().next().toFieldList());
	}
	gProps.clearUniqueFieldCombinations();
	return gProps;
}

Example #30

Source File: CoGroupRawDescriptor.java From flink with Apache License 2.0

5 votes

@Override
public boolean areCompatible(RequestedGlobalProperties requested1, RequestedGlobalProperties requested2,
		GlobalProperties produced1, GlobalProperties produced2)
{
	return produced1.getPartitioning() == produced2.getPartitioning() && 
			(produced1.getCustomPartitioner() == null ? 
				produced2.getCustomPartitioner() == null :
				produced1.getCustomPartitioner().equals(produced2.getCustomPartitioner()));
}