Java Code Examples for org.apache.flink.optimizer.plan.OptimizedPlan#getDataSinks()

The following examples show how to use org.apache.flink.optimizer.plan.OptimizedPlan#getDataSinks() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JavaApiPostPass.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public void postPass(OptimizedPlan plan) {

	executionConfig = plan.getOriginalPlan().getExecutionConfig();

	for (SinkPlanNode sink : plan.getDataSinks()) {
		traverse(sink);
	}
}

Example 2

Source File: PlanJSONDumpGenerator.java From Flink-CEPplus with Apache License 2.0

5 votes

public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) {
	Collection<SinkPlanNode> sinks = plan.getDataSinks();
	if (sinks instanceof List) {
		dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer);
	} else {
		List<SinkPlanNode> n = new ArrayList<SinkPlanNode>();
		n.addAll(sinks);
		dumpOptimizerPlanAsJSON(n, writer);
	}
}

Example 3

Source File: JavaApiPostPass.java From flink with Apache License 2.0

5 votes

@Override
public void postPass(OptimizedPlan plan) {

	executionConfig = plan.getOriginalPlan().getExecutionConfig();

	for (SinkPlanNode sink : plan.getDataSinks()) {
		traverse(sink);
	}
}

Example 4

Source File: PlanJSONDumpGenerator.java From flink with Apache License 2.0

5 votes

public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) {
	Collection<SinkPlanNode> sinks = plan.getDataSinks();
	if (sinks instanceof List) {
		dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer);
	} else {
		List<SinkPlanNode> n = new ArrayList<SinkPlanNode>();
		n.addAll(sinks);
		dumpOptimizerPlanAsJSON(n, writer);
	}
}

Example 5

Source File: JavaApiPostPass.java From flink with Apache License 2.0

5 votes

@Override
public void postPass(OptimizedPlan plan) {

	executionConfig = plan.getOriginalPlan().getExecutionConfig();

	for (SinkPlanNode sink : plan.getDataSinks()) {
		traverse(sink);
	}
}

Example 6

Source File: PlanJSONDumpGenerator.java From flink with Apache License 2.0

5 votes

public void dumpOptimizerPlanAsJSON(OptimizedPlan plan, PrintWriter writer) {
	Collection<SinkPlanNode> sinks = plan.getDataSinks();
	if (sinks instanceof List) {
		dumpOptimizerPlanAsJSON((List<SinkPlanNode>) sinks, writer);
	} else {
		List<SinkPlanNode> n = new ArrayList<SinkPlanNode>();
		n.addAll(sinks);
		dumpOptimizerPlanAsJSON(n, writer);
	}
}

Example 7

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *                (SRC A)  
 *                   |
 *                (MAP A)
 *             /         \   
 *          (MAP B)      (MAP C)
 *           /           /     \
 *        (SINK A)    (SINK B)  (SINK C)
 * </pre>
 */
@SuppressWarnings("unchecked")
@Test
public void testBranchingWithMultipleDataSinks2() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Long> source = env.generateSequence(1, 10000);

		DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
		DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
		DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());

		mappedB.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());

		Plan plan = env.createProgramPlan();
		Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());

		OptimizedPlan oPlan = compileNoStats(plan);

		// ---------- check the optimizer plan ----------

		// number of sinks
		assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());

		// remove matching sinks to check relation
		for (SinkPlanNode sink : oPlan.getDataSinks()) {
			assertTrue(sinks.remove(sink.getProgramOperator()));
		}
		assertTrue(sinks.isEmpty());

		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 8

Source File: BranchingPlansCompilerTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *             (SRC A)     
 *             /     \      
 *        (SINK A)    (SINK B)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinksSmall() {
	try {
		String outPath1 = "/tmp/out1";
		String outPath2 = "/tmp/out2";

		// construct the plan
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);
		DataSet<Long> source1 = env.generateSequence(0,1);

		source1.writeAsText(outPath1);
		source1.writeAsText(outPath2);

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		
		// ---------- check the optimizer plan ----------
		
		// number of sinks
		Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
		
		// sinks contain all sink paths
		Set<String> allSinks = new HashSet<String>();
		allSinks.add(outPath1);
		allSinks.add(outPath2);
		
		for (SinkPlanNode n : oPlan.getDataSinks()) {
			String path = ((TextOutputFormat<String>)n.getSinkNode().getOperator()
					.getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString();
			Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
		}
		
		// ---------- compile plan to job graph to verify that no error is thrown ----------
		
		JobGraphGenerator jobGen = new JobGraphGenerator();
		jobGen.compileJobGraph(oPlan);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example 9

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *                (SRC A)  
 *                   |
 *                (MAP A)
 *             /         \   
 *          (MAP B)      (MAP C)
 *           /           /     \
 *        (SINK A)    (SINK B)  (SINK C)
 * </pre>
 */
@SuppressWarnings("unchecked")
@Test
public void testBranchingWithMultipleDataSinks2() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Long> source = env.generateSequence(1, 10000);

		DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
		DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
		DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());

		mappedB.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());

		Plan plan = env.createProgramPlan();
		Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());

		OptimizedPlan oPlan = compileNoStats(plan);

		// ---------- check the optimizer plan ----------

		// number of sinks
		assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());

		// remove matching sinks to check relation
		for (SinkPlanNode sink : oPlan.getDataSinks()) {
			assertTrue(sinks.remove(sink.getProgramOperator()));
		}
		assertTrue(sinks.isEmpty());

		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 10

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *             (SRC A)     
 *             /     \      
 *        (SINK A)    (SINK B)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinksSmall() {
	try {
		String outPath1 = "/tmp/out1";
		String outPath2 = "/tmp/out2";

		// construct the plan
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);
		DataSet<Long> source1 = env.generateSequence(0,1);

		source1.writeAsText(outPath1);
		source1.writeAsText(outPath2);

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		
		// ---------- check the optimizer plan ----------
		
		// number of sinks
		Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
		
		// sinks contain all sink paths
		Set<String> allSinks = new HashSet<String>();
		allSinks.add(outPath1);
		allSinks.add(outPath2);
		
		for (SinkPlanNode n : oPlan.getDataSinks()) {
			String path = ((TextOutputFormat<String>)n.getSinkNode().getOperator()
					.getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString();
			Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
		}
		
		// ---------- compile plan to job graph to verify that no error is thrown ----------
		
		JobGraphGenerator jobGen = new JobGraphGenerator();
		jobGen.compileJobGraph(oPlan);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}

Example 11

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *                (SRC A)  
 *                   |
 *                (MAP A)
 *             /         \   
 *          (MAP B)      (MAP C)
 *           /           /     \
 *        (SINK A)    (SINK B)  (SINK C)
 * </pre>
 */
@SuppressWarnings("unchecked")
@Test
public void testBranchingWithMultipleDataSinks2() {
	try {
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);

		DataSet<Long> source = env.generateSequence(1, 10000);

		DataSet<Long> mappedA = source.map(new IdentityMapper<Long>());
		DataSet<Long> mappedB = mappedA.map(new IdentityMapper<Long>());
		DataSet<Long> mappedC = mappedA.map(new IdentityMapper<Long>());

		mappedB.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());
		mappedC.output(new DiscardingOutputFormat<Long>());

		Plan plan = env.createProgramPlan();
		Set<Operator<?>> sinks = new HashSet<Operator<?>>(plan.getDataSinks());

		OptimizedPlan oPlan = compileNoStats(plan);

		// ---------- check the optimizer plan ----------

		// number of sinks
		assertEquals("Wrong number of data sinks.", 3, oPlan.getDataSinks().size());

		// remove matching sinks to check relation
		for (SinkPlanNode sink : oPlan.getDataSinks()) {
			assertTrue(sinks.remove(sink.getProgramOperator()));
		}
		assertTrue(sinks.isEmpty());

		new JobGraphGenerator().compileJobGraph(oPlan);
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 12

Source File: BranchingPlansCompilerTest.java From flink with Apache License 2.0

4 votes

/**
 * 
 * <pre>
 *             (SRC A)     
 *             /     \      
 *        (SINK A)    (SINK B)
 * </pre>
 */
@Test
public void testBranchingWithMultipleDataSinksSmall() {
	try {
		String outPath1 = "/tmp/out1";
		String outPath2 = "/tmp/out2";

		// construct the plan
		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(DEFAULT_PARALLELISM);
		DataSet<Long> source1 = env.generateSequence(0,1);

		source1.writeAsText(outPath1);
		source1.writeAsText(outPath2);

		Plan plan = env.createProgramPlan();
		OptimizedPlan oPlan = compileNoStats(plan);
		
		// ---------- check the optimizer plan ----------
		
		// number of sinks
		Assert.assertEquals("Wrong number of data sinks.", 2, oPlan.getDataSinks().size());
		
		// sinks contain all sink paths
		Set<String> allSinks = new HashSet<String>();
		allSinks.add(outPath1);
		allSinks.add(outPath2);
		
		for (SinkPlanNode n : oPlan.getDataSinks()) {
			String path = ((TextOutputFormat<String>)n.getSinkNode().getOperator()
					.getFormatWrapper().getUserCodeObject()).getOutputFilePath().toString();
			Assert.assertTrue("Invalid data sink.", allSinks.remove(path));
		}
		
		// ---------- compile plan to job graph to verify that no error is thrown ----------
		
		JobGraphGenerator jobGen = new JobGraphGenerator();
		jobGen.compileJobGraph(oPlan);
	} catch (Exception e) {
		e.printStackTrace();
		Assert.fail(e.getMessage());
	}
}