org.apache.flink.optimizer.plan.NAryUnionPlanNode Java Examples

The following examples show how to use org.apache.flink.optimizer.plan.NAryUnionPlanNode. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: BinaryUnionReplacer.java From Flink-CEPplus with Apache License 2.0

6 votes

public void collect(Channel in, List<Channel> inputs) {
	if (in.getSource() instanceof NAryUnionPlanNode) {
		// sanity check
		if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
			throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
		}
		if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
			throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
		}

		inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
	} else {
		// is not a collapsed union node, so we take the channel directly
		inputs.add(in);
	}
}

Example #2

Source File: BinaryUnionReplacer.java From flink with Apache License 2.0

6 votes

public void collect(Channel in, List<Channel> inputs) {
	if (in.getSource() instanceof NAryUnionPlanNode) {
		// sanity check
		if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
			throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
		}
		if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
			throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
		}

		inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
	} else {
		// is not a collapsed union node, so we take the channel directly
		inputs.add(in);
	}
}

Example #3

Source File: BinaryUnionReplacer.java From flink with Apache License 2.0

6 votes

public void collect(Channel in, List<Channel> inputs) {
	if (in.getSource() instanceof NAryUnionPlanNode) {
		// sanity check
		if (in.getShipStrategy() != ShipStrategyType.FORWARD) {
			throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators.");
		}
		if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) {
			throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators.");
		}

		inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs());
	} else {
		// is not a collapsed union node, so we take the channel directly
		inputs.add(in);
	}
}

Example #4

Source File: IterationCompilerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testWorksetIterationWithUnionRoot() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(43);
		
		DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
				.map(new MapFunction<Long, Tuple2<Long, Long>>() {
					@Override
					public Tuple2<Long, Long> map(Long value){ return null; }
				});
				
				
		DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
		iter.closeWith(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			, iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
					iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			)
		.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
		
		// make sure that the root is part of the dynamic path
		
		// the "NoOp"a that come after the union.
		SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
		SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
		
		NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
		NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
		
		assertTrue(nextWorksetNoop.isOnDynamicPath());
		assertTrue(nextWorksetNoop.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaNoop.isOnDynamicPath());
		assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
		
		assertTrue(nextWorksetUnion.isOnDynamicPath());
		assertTrue(nextWorksetUnion.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaUnion.isOnDynamicPath());
		assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #5

Source File: UnionReplacementTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Test the input and output shipping strategies for union operators with input and output
 * operators with different parallelisms.
 *
 * Src1 - Map(fullP) -\-/- Union - Map(fullP) - Out
 *                     X
 * Src2 - Map(halfP) -/-\- Union - Map(halfP) - Out
 *
 * The union operator must always have the same parallelism as its successor and connect to it
 * with a FORWARD strategy.
 * In this program, the input connections for union should be FORWARD for parallelism-preserving
 * connections and PARTITION_RANDOM for parallelism-changing connections.
 *
 */
@Test
public void testUnionInputOutputDifferentDOP() throws Exception {

	int fullDop = DEFAULT_PARALLELISM;
	int halfDop = DEFAULT_PARALLELISM / 2;

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSet<Tuple2<Long, Long>> in1 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("inDopFull");
	DataSet<Tuple2<Long, Long>> in2 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("inDopHalf");

	DataSet<Tuple2<Long, Long>> union = in1.union(in2);

	DataSet<Tuple2<Long, Long>> dopFullMap = union
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("outDopFull");
	DataSet<Tuple2<Long, Long>> dopHalfMap = union
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("outDopHalf");

	dopFullMap.output(new DiscardingOutputFormat<>());
	dopHalfMap.output(new DiscardingOutputFormat<>());

	// -----------------------------------------------------------------------------------------
	// Verify optimized plan
	// -----------------------------------------------------------------------------------------

	OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());

	OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(optimizedPlan);

	SingleInputPlanNode inDopFull = resolver.getNode("inDopFull");
	SingleInputPlanNode inDopHalf = resolver.getNode("inDopHalf");
	SingleInputPlanNode outDopFull = resolver.getNode("outDopFull");
	SingleInputPlanNode outDopHalf = resolver.getNode("outDopHalf");
	NAryUnionPlanNode unionDopFull = (NAryUnionPlanNode) outDopFull.getInput().getSource();
	NAryUnionPlanNode unionDopHalf = (NAryUnionPlanNode) outDopHalf.getInput().getSource();

	// check in map nodes
	assertEquals(2, inDopFull.getOutgoingChannels().size());
	assertEquals(2, inDopHalf.getOutgoingChannels().size());
	assertEquals(fullDop, inDopFull.getParallelism());
	assertEquals(halfDop, inDopHalf.getParallelism());

	// check union nodes
	assertEquals(fullDop, unionDopFull.getParallelism());
	assertEquals(halfDop, unionDopHalf.getParallelism());

	// check out map nodes
	assertEquals(fullDop, outDopFull.getParallelism());
	assertEquals(halfDop, outDopHalf.getParallelism());

	// check Union -> outMap ship strategies
	assertEquals(ShipStrategyType.FORWARD, outDopHalf.getInput().getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, outDopFull.getInput().getShipStrategy());

	// check inMap -> Union ship strategies
	Channel fullFull;
	Channel fullHalf;
	Channel halfFull;
	Channel halfHalf;

	if (inDopFull.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		fullFull = inDopFull.getOutgoingChannels().get(0);
		fullHalf = inDopFull.getOutgoingChannels().get(1);
	} else {
		fullFull = inDopFull.getOutgoingChannels().get(1);
		fullHalf = inDopFull.getOutgoingChannels().get(0);
	}
	if (inDopHalf.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		halfFull = inDopHalf.getOutgoingChannels().get(0);
		halfHalf = inDopHalf.getOutgoingChannels().get(1);
	} else {
		halfFull = inDopHalf.getOutgoingChannels().get(1);
		halfHalf = inDopHalf.getOutgoingChannels().get(0);
	}

	assertEquals(ShipStrategyType.FORWARD, fullFull.getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, halfHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, fullHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, halfFull.getShipStrategy());
}

Example #6

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources1() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}

Example #7

Source File: PropertyDataSourceTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources2() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byCountry", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}

Example #8

Source File: IterationCompilerTest.java From flink with Apache License 2.0

4 votes

@Test
public void testWorksetIterationWithUnionRoot() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(43);
		
		DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
				.map(new MapFunction<Long, Tuple2<Long, Long>>() {
					@Override
					public Tuple2<Long, Long> map(Long value){ return null; }
				});
				
				
		DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
		iter.closeWith(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			, iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
					iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			)
		.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
		
		// make sure that the root is part of the dynamic path
		
		// the "NoOp"a that come after the union.
		SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
		SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
		
		NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
		NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
		
		assertTrue(nextWorksetNoop.isOnDynamicPath());
		assertTrue(nextWorksetNoop.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaNoop.isOnDynamicPath());
		assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
		
		assertTrue(nextWorksetUnion.isOnDynamicPath());
		assertTrue(nextWorksetUnion.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaUnion.isOnDynamicPath());
		assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #9

Source File: UnionReplacementTest.java From flink with Apache License 2.0

4 votes

/**
 * Test the input and output shipping strategies for union operators with input and output
 * operators with different parallelisms.
 *
 * Src1 - Map(fullP) -\-/- Union - Map(fullP) - Out
 *                     X
 * Src2 - Map(halfP) -/-\- Union - Map(halfP) - Out
 *
 * The union operator must always have the same parallelism as its successor and connect to it
 * with a FORWARD strategy.
 * In this program, the input connections for union should be FORWARD for parallelism-preserving
 * connections and PARTITION_RANDOM for parallelism-changing connections.
 *
 */
@Test
public void testUnionInputOutputDifferentDOP() throws Exception {

	int fullDop = DEFAULT_PARALLELISM;
	int halfDop = DEFAULT_PARALLELISM / 2;

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSet<Tuple2<Long, Long>> in1 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("inDopFull");
	DataSet<Tuple2<Long, Long>> in2 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("inDopHalf");

	DataSet<Tuple2<Long, Long>> union = in1.union(in2);

	DataSet<Tuple2<Long, Long>> dopFullMap = union
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("outDopFull");
	DataSet<Tuple2<Long, Long>> dopHalfMap = union
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("outDopHalf");

	dopFullMap.output(new DiscardingOutputFormat<>());
	dopHalfMap.output(new DiscardingOutputFormat<>());

	// -----------------------------------------------------------------------------------------
	// Verify optimized plan
	// -----------------------------------------------------------------------------------------

	OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());

	OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(optimizedPlan);

	SingleInputPlanNode inDopFull = resolver.getNode("inDopFull");
	SingleInputPlanNode inDopHalf = resolver.getNode("inDopHalf");
	SingleInputPlanNode outDopFull = resolver.getNode("outDopFull");
	SingleInputPlanNode outDopHalf = resolver.getNode("outDopHalf");
	NAryUnionPlanNode unionDopFull = (NAryUnionPlanNode) outDopFull.getInput().getSource();
	NAryUnionPlanNode unionDopHalf = (NAryUnionPlanNode) outDopHalf.getInput().getSource();

	// check in map nodes
	assertEquals(2, inDopFull.getOutgoingChannels().size());
	assertEquals(2, inDopHalf.getOutgoingChannels().size());
	assertEquals(fullDop, inDopFull.getParallelism());
	assertEquals(halfDop, inDopHalf.getParallelism());

	// check union nodes
	assertEquals(fullDop, unionDopFull.getParallelism());
	assertEquals(halfDop, unionDopHalf.getParallelism());

	// check out map nodes
	assertEquals(fullDop, outDopFull.getParallelism());
	assertEquals(halfDop, outDopHalf.getParallelism());

	// check Union -> outMap ship strategies
	assertEquals(ShipStrategyType.FORWARD, outDopHalf.getInput().getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, outDopFull.getInput().getShipStrategy());

	// check inMap -> Union ship strategies
	Channel fullFull;
	Channel fullHalf;
	Channel halfFull;
	Channel halfHalf;

	if (inDopFull.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		fullFull = inDopFull.getOutgoingChannels().get(0);
		fullHalf = inDopFull.getOutgoingChannels().get(1);
	} else {
		fullFull = inDopFull.getOutgoingChannels().get(1);
		fullHalf = inDopFull.getOutgoingChannels().get(0);
	}
	if (inDopHalf.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		halfFull = inDopHalf.getOutgoingChannels().get(0);
		halfHalf = inDopHalf.getOutgoingChannels().get(1);
	} else {
		halfFull = inDopHalf.getOutgoingChannels().get(1);
		halfHalf = inDopHalf.getOutgoingChannels().get(0);
	}

	assertEquals(ShipStrategyType.FORWARD, fullFull.getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, halfHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, fullHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, halfFull.getShipStrategy());
}

Example #10

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources1() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}

Example #11

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources2() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byCountry", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}

Example #12

Source File: IterationCompilerTest.java From flink with Apache License 2.0

4 votes

@Test
public void testWorksetIterationWithUnionRoot() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(43);
		
		DataSet<Tuple2<Long, Long>> input = env.generateSequence(1, 20)
				.map(new MapFunction<Long, Tuple2<Long, Long>>() {
					@Override
					public Tuple2<Long, Long> map(Long value){ return null; }
				});
				
				
		DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iter = input.iterateDelta(input, 100, 0);
		iter.closeWith(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
				iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			, iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>())
			.union(
					iter.getWorkset().map(new IdentityMapper<Tuple2<Long,Long>>()))
			)
		.output(new DiscardingOutputFormat<Tuple2<Long, Long>>());
		
		Plan p = env.createProgramPlan();
		OptimizedPlan op = compileNoStats(p);
		
		SinkPlanNode sink = op.getDataSinks().iterator().next();
		WorksetIterationPlanNode iterNode = (WorksetIterationPlanNode) sink.getInput().getSource();
		
		// make sure that the root is part of the dynamic path
		
		// the "NoOp"a that come after the union.
		SingleInputPlanNode nextWorksetNoop = (SingleInputPlanNode) iterNode.getNextWorkSetPlanNode();
		SingleInputPlanNode solutionDeltaNoop = (SingleInputPlanNode) iterNode.getSolutionSetDeltaPlanNode();
		
		NAryUnionPlanNode nextWorksetUnion = (NAryUnionPlanNode) nextWorksetNoop.getInput().getSource();
		NAryUnionPlanNode solutionDeltaUnion = (NAryUnionPlanNode) solutionDeltaNoop.getInput().getSource();
		
		assertTrue(nextWorksetNoop.isOnDynamicPath());
		assertTrue(nextWorksetNoop.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaNoop.isOnDynamicPath());
		assertTrue(solutionDeltaNoop.getCostWeight() >= 1);
		
		assertTrue(nextWorksetUnion.isOnDynamicPath());
		assertTrue(nextWorksetUnion.getCostWeight() >= 1);
		
		assertTrue(solutionDeltaUnion.isOnDynamicPath());
		assertTrue(solutionDeltaUnion.getCostWeight() >= 1);
		
		new JobGraphGenerator().compileJobGraph(op);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example #13

Source File: UnionReplacementTest.java From flink with Apache License 2.0

4 votes

/**
 * Test the input and output shipping strategies for union operators with input and output
 * operators with different parallelisms.
 *
 * Src1 - Map(fullP) -\-/- Union - Map(fullP) - Out
 *                     X
 * Src2 - Map(halfP) -/-\- Union - Map(halfP) - Out
 *
 * The union operator must always have the same parallelism as its successor and connect to it
 * with a FORWARD strategy.
 * In this program, the input connections for union should be FORWARD for parallelism-preserving
 * connections and PARTITION_RANDOM for parallelism-changing connections.
 *
 */
@Test
public void testUnionInputOutputDifferentDOP() throws Exception {

	int fullDop = DEFAULT_PARALLELISM;
	int halfDop = DEFAULT_PARALLELISM / 2;

	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSet<Tuple2<Long, Long>> in1 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("inDopFull");
	DataSet<Tuple2<Long, Long>> in2 = env.fromElements(new Tuple2<>(0L, 0L))
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("inDopHalf");

	DataSet<Tuple2<Long, Long>> union = in1.union(in2);

	DataSet<Tuple2<Long, Long>> dopFullMap = union
		.map(new IdentityMapper<>()).setParallelism(fullDop).name("outDopFull");
	DataSet<Tuple2<Long, Long>> dopHalfMap = union
		.map(new IdentityMapper<>()).setParallelism(halfDop).name("outDopHalf");

	dopFullMap.output(new DiscardingOutputFormat<>());
	dopHalfMap.output(new DiscardingOutputFormat<>());

	// -----------------------------------------------------------------------------------------
	// Verify optimized plan
	// -----------------------------------------------------------------------------------------

	OptimizedPlan optimizedPlan = compileNoStats(env.createProgramPlan());

	OptimizerPlanNodeResolver resolver = getOptimizerPlanNodeResolver(optimizedPlan);

	SingleInputPlanNode inDopFull = resolver.getNode("inDopFull");
	SingleInputPlanNode inDopHalf = resolver.getNode("inDopHalf");
	SingleInputPlanNode outDopFull = resolver.getNode("outDopFull");
	SingleInputPlanNode outDopHalf = resolver.getNode("outDopHalf");
	NAryUnionPlanNode unionDopFull = (NAryUnionPlanNode) outDopFull.getInput().getSource();
	NAryUnionPlanNode unionDopHalf = (NAryUnionPlanNode) outDopHalf.getInput().getSource();

	// check in map nodes
	assertEquals(2, inDopFull.getOutgoingChannels().size());
	assertEquals(2, inDopHalf.getOutgoingChannels().size());
	assertEquals(fullDop, inDopFull.getParallelism());
	assertEquals(halfDop, inDopHalf.getParallelism());

	// check union nodes
	assertEquals(fullDop, unionDopFull.getParallelism());
	assertEquals(halfDop, unionDopHalf.getParallelism());

	// check out map nodes
	assertEquals(fullDop, outDopFull.getParallelism());
	assertEquals(halfDop, outDopHalf.getParallelism());

	// check Union -> outMap ship strategies
	assertEquals(ShipStrategyType.FORWARD, outDopHalf.getInput().getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, outDopFull.getInput().getShipStrategy());

	// check inMap -> Union ship strategies
	Channel fullFull;
	Channel fullHalf;
	Channel halfFull;
	Channel halfHalf;

	if (inDopFull.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		fullFull = inDopFull.getOutgoingChannels().get(0);
		fullHalf = inDopFull.getOutgoingChannels().get(1);
	} else {
		fullFull = inDopFull.getOutgoingChannels().get(1);
		fullHalf = inDopFull.getOutgoingChannels().get(0);
	}
	if (inDopHalf.getOutgoingChannels().get(0).getTarget() == unionDopFull) {
		halfFull = inDopHalf.getOutgoingChannels().get(0);
		halfHalf = inDopHalf.getOutgoingChannels().get(1);
	} else {
		halfFull = inDopHalf.getOutgoingChannels().get(1);
		halfHalf = inDopHalf.getOutgoingChannels().get(0);
	}

	assertEquals(ShipStrategyType.FORWARD, fullFull.getShipStrategy());
	assertEquals(ShipStrategyType.FORWARD, halfHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, fullHalf.getShipStrategy());
	assertEquals(ShipStrategyType.PARTITION_RANDOM, halfFull.getShipStrategy());
}

Example #14

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources1() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}

Example #15

Source File: PropertyDataSourceTest.java From flink with Apache License 2.0

4 votes

@Test
public void checkCoPartitionedSources2() {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.setParallelism(DEFAULT_PARALLELISM);

	DataSource<Tuple2<Long, String>> data1 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data1.getSplitDataProperties()
			.splitsPartitionedBy("byCountry", 0);

	DataSource<Tuple2<Long, String>> data2 =
			env.readCsvFile("/some/path").types(Long.class, String.class);

	data2.getSplitDataProperties()
			.splitsPartitionedBy("byDate", 0);

	data1.union(data2).output(new DiscardingOutputFormat<Tuple2<Long, String>>());

	Plan plan = env.createProgramPlan();

	// submit the plan to the compiler
	OptimizedPlan oPlan = compileNoStats(plan);

	// check the optimized Plan
	SinkPlanNode sinkNode = oPlan.getDataSinks().iterator().next();
	SourcePlanNode sourceNode1 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(0).getSource();
	SourcePlanNode sourceNode2 = (SourcePlanNode) ((NAryUnionPlanNode)sinkNode.getPredecessor()).getListOfInputs().get(1).getSource();

	GlobalProperties gprops1 = sourceNode1.getGlobalProperties();
	LocalProperties lprops1 = sourceNode1.getLocalProperties();
	GlobalProperties gprops2 = sourceNode2.getGlobalProperties();
	LocalProperties lprops2 = sourceNode2.getLocalProperties();

	Assert.assertTrue((new FieldSet(gprops1.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops1.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops1.getGroupedFields() == null);
	Assert.assertTrue(lprops1.getOrdering() == null);

	Assert.assertTrue((new FieldSet(gprops2.getPartitioningFields().toArray())).equals(new FieldSet(0)));
	Assert.assertTrue(gprops2.getPartitioning() == PartitioningProperty.CUSTOM_PARTITIONING);
	Assert.assertTrue(lprops2.getGroupedFields() == null);
	Assert.assertTrue(lprops2.getOrdering() == null);

	Assert.assertTrue(!gprops1.getCustomPartitioner().equals(gprops2.getCustomPartitioner()));
}