org.apache.flink.optimizer.dag.DataSinkNode Java Examples

private static void verifyPlanDump(Class<?> entrypoint, String... args) throws Exception {
	final PackagedProgram program = PackagedProgram
		.newBuilder()
		.setEntryPointClassName(entrypoint.getName())
		.setArguments(args)
		.build();

	final Pipeline pipeline = PackagedProgramUtils.getPipelineFromProgram(program, new Configuration(), 1, true);

	assertTrue(pipeline instanceof Plan);

	final Plan plan = (Plan) pipeline;

	final List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
	final PlanJSONDumpGenerator dumper = new PlanJSONDumpGenerator();
	final String json = dumper.getPactPlanAsJSON(sinks);

	try (JsonParser parser = new JsonFactory().createParser(json)) {
		while (parser.nextToken() != null) {
		}
	}
}
 

public DataSinkNode getSinkNode() {
	if (this.template instanceof DataSinkNode) {
		return (DataSinkNode) this.template;
	} else {
		throw new RuntimeException();
	}
}
 

/**
 * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that
 * local sorting and range partitioning are handled by the incoming channel already.
 * 
 * @param template The template optimizer node that this candidate is created for.
 */
public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) {
	super(template, nodeName, input, DriverStrategy.NONE);
	
	this.globalProps = input.getGlobalProperties().clone();
	this.localProps = input.getLocalProperties().clone();
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

/**
 * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that
 * local sorting and range partitioning are handled by the incoming channel already.
 * 
 * @param template The template optimizer node that this candidate is created for.
 */
public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) {
	super(template, nodeName, input, DriverStrategy.NONE);
	
	this.globalProps = input.getGlobalProperties().clone();
	this.localProps = input.getLocalProperties().clone();
}
 

public DataSinkNode getSinkNode() {
	if (this.template instanceof DataSinkNode) {
		return (DataSinkNode) this.template;
	} else {
		throw new RuntimeException();
	}
}
 

public DataSinkNode getSinkNode() {
	if (this.template instanceof DataSinkNode) {
		return (DataSinkNode) this.template;
	} else {
		throw new RuntimeException();
	}
}
 

/**
 * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that
 * local sorting and range partitioning are handled by the incoming channel already.
 * 
 * @param template The template optimizer node that this candidate is created for.
 */
public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) {
	super(template, nodeName, input, DriverStrategy.NONE);
	
	this.globalProps = input.getGlobalProperties().clone();
	this.localProps = input.getLocalProperties().clone();
}
 

private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism,
								ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) {
	if (closure == null){
		con2node = new HashMap<Operator<?>, OptimizerNode>();
	} else {
		con2node = closure;
	}

	this.sinks = new ArrayList<DataSinkNode>(2);
	this.defaultParallelism = defaultParallelism;
	this.parent = parent;
	this.defaultDataExchangeMode = dataExchangeMode;
	this.forceParallelism = forceParallelism;
}
 

/**
 * Returns the analyzed plan without any optimizations.
 *
 * @return
 *         the analyzed plan without any optimizations.
 * @throws ProgramInvocationException Thrown if an error occurred in the
 *  user-provided pact assembler. This may indicate
 *         missing parameters for generation.
 */
public String getPreviewPlan() throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(this.getUserCodeClassLoader());
	List<DataSinkNode> previewPlan;

	if (isUsingProgramEntryPoint()) {
		previewPlan = Optimizer.createPreOptimizedPlan(getPlan());
	}
	else if (isUsingInteractiveMode()) {
		// temporary hack to support the web client
		PreviewPlanEnvironment env = new PreviewPlanEnvironment();
		env.setAsContext();
		try {
			invokeInteractiveModeForExecution();
		}
		catch (ProgramInvocationException e) {
			throw e;
		}
		catch (Throwable t) {
			// the invocation gets aborted with the preview plan
			if (env.previewPlan == null) {
				if (env.preview != null) {
					return env.preview;
				} else {
					throw new ProgramInvocationException("The program caused an error: ", getPlan().getJobId(), t);
				}
			}
		}
		finally {
			env.unsetAsContext();
		}

		if (env.previewPlan != null) {
			previewPlan =  env.previewPlan;
		} else {
			throw new ProgramInvocationException(
				"The program plan could not be fetched. The program silently swallowed the control flow exceptions.",
				getPlan().getJobId());
		}
	}
	else {
		throw new RuntimeException();
	}

	PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
	StringWriter string = new StringWriter(1024);
	try (PrintWriter pw = new PrintWriter(string)) {
		jsonGen.dumpPactPlanAsJSON(previewPlan, pw);
	}
	return string.toString();

}
 

public List<DataSinkNode> getSinks() {
	return sinks;
}
 

public String getPactPlanAsJSON(List<DataSinkNode> nodes) {
	StringWriter sw = new StringWriter();
	PrintWriter pw = new PrintWriter(sw);
	dumpPactPlanAsJSON(nodes, pw);
	return sw.toString();
}
 

public void dumpPactPlanAsJSON(List<DataSinkNode> nodes, PrintWriter writer) {
	@SuppressWarnings("unchecked")
	List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
	compilePlanToJSON(n, writer);
}
 

public List<DataSinkNode> getSinks() {
	return sinks;
}
 

public String getPactPlanAsJSON(List<DataSinkNode> nodes) {
	StringWriter sw = new StringWriter();
	PrintWriter pw = new PrintWriter(sw);
	dumpPactPlanAsJSON(nodes, pw);
	return sw.toString();
}
 

public void dumpPactPlanAsJSON(List<DataSinkNode> nodes, PrintWriter writer) {
	@SuppressWarnings("unchecked")
	List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
	compilePlanToJSON(n, writer);
}
 

/**
 * Returns the analyzed plan without any optimizations.
 *
 * @return
 *         the analyzed plan without any optimizations.
 * @throws ProgramInvocationException Thrown if an error occurred in the
 *  user-provided pact assembler. This may indicate
 *         missing parameters for generation.
 */
public String getPreviewPlan() throws ProgramInvocationException {
	Thread.currentThread().setContextClassLoader(this.getUserCodeClassLoader());
	List<DataSinkNode> previewPlan;

	if (isUsingProgramEntryPoint()) {
		previewPlan = Optimizer.createPreOptimizedPlan(getPlan());
	}
	else if (isUsingInteractiveMode()) {
		// temporary hack to support the web client
		PreviewPlanEnvironment env = new PreviewPlanEnvironment();
		env.setAsContext();
		try {
			invokeInteractiveModeForExecution();
		}
		catch (ProgramInvocationException e) {
			throw e;
		}
		catch (Throwable t) {
			// the invocation gets aborted with the preview plan
			if (env.previewPlan == null) {
				if (env.preview != null) {
					return env.preview;
				} else {
					throw new ProgramInvocationException("The program caused an error: ", getPlan().getJobId(), t);
				}
			}
		}
		finally {
			env.unsetAsContext();
		}

		if (env.previewPlan != null) {
			previewPlan =  env.previewPlan;
		} else {
			throw new ProgramInvocationException(
				"The program plan could not be fetched. The program silently swallowed the control flow exceptions.",
				getPlan().getJobId());
		}
	}
	else {
		throw new RuntimeException();
	}

	PlanJSONDumpGenerator jsonGen = new PlanJSONDumpGenerator();
	StringWriter string = new StringWriter(1024);
	try (PrintWriter pw = new PrintWriter(string)) {
		jsonGen.dumpPactPlanAsJSON(previewPlan, pw);
	}
	return string.toString();

}
 

public List<DataSinkNode> getSinks() {
	return sinks;
}
 

public String getPactPlanAsJSON(List<DataSinkNode> nodes) {
	StringWriter sw = new StringWriter();
	PrintWriter pw = new PrintWriter(sw);
	dumpPactPlanAsJSON(nodes, pw);
	return sw.toString();
}
 

public void dumpPactPlanAsJSON(List<DataSinkNode> nodes, PrintWriter writer) {
	@SuppressWarnings("unchecked")
	List<DumpableNode<?>> n = (List<DumpableNode<?>>) (List<?>) nodes;
	compilePlanToJSON(n, writer);
}
 

/**
 * This function performs only the first step to the compilation process - the creation of the optimizer
 * representation of the plan. No estimations or enumerations of alternatives are done here.
 * 
 * @param program The plan to generate the optimizer representation for.
 * @return The optimizer representation of the plan, as a collection of all data sinks
 *         from the plan can be traversed.
 */
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
	GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
	program.accept(graphCreator);
	return graphCreator.getSinks();
}
 

/**
 * Creates a JSON representation of the given dataflow plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow plan (prior to optimization) as a JSON string.
 */
public static String getPlanAsJSON(Plan plan) {
	List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
	return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks);
}
 

/**
 * This function performs only the first step to the compilation process - the creation of the optimizer
 * representation of the plan. No estimations or enumerations of alternatives are done here.
 * 
 * @param program The plan to generate the optimizer representation for.
 * @return The optimizer representation of the plan, as a collection of all data sinks
 *         from the plan can be traversed.
 */
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
	GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
	program.accept(graphCreator);
	return graphCreator.getSinks();
}
 

/**
 * This function performs only the first step to the compilation process - the creation of the optimizer
 * representation of the plan. No estimations or enumerations of alternatives are done here.
 * 
 * @param program The plan to generate the optimizer representation for.
 * @return The optimizer representation of the plan, as a collection of all data sinks
 *         from the plan can be traversed.
 */
public static List<DataSinkNode> createPreOptimizedPlan(Plan program) {
	GraphCreatingVisitor graphCreator = new GraphCreatingVisitor(1, null);
	program.accept(graphCreator);
	return graphCreator.getSinks();
}
 

/**
 * Creates a JSON representation of the given dataflow plan.
 *
 * @param plan The dataflow plan.
 * @return The dataflow plan (prior to optimization) as a JSON string.
 */
public static String getPlanAsJSON(Plan plan) {
	List<DataSinkNode> sinks = Optimizer.createPreOptimizedPlan(plan);
	return new PlanJSONDumpGenerator().getPactPlanAsJSON(sinks);
}