org.apache.flink.api.common.ExecutionMode Java Examples
The following examples show how to use
org.apache.flink.api.common.ExecutionMode.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DataExchangeModeClosedBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.PIPELINED, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.PIPELINED, // to flatMap DataExchangeMode.PIPELINED, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.PIPELINED // to sink after coGroup ); }
Example #2
Source File: OptimizerNode.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * This function connects the operators that produce the broadcast inputs to this operator. * * @param operatorToNode The map from program operators to optimizer nodes. * @param defaultExchangeMode The data exchange mode to use, if the operator does not * specify one. * * @throws CompilerException */ public void setBroadcastInputs(Map<Operator<?>, OptimizerNode> operatorToNode, ExecutionMode defaultExchangeMode) { // skip for Operators that don't support broadcast variables if (!(getOperator() instanceof AbstractUdfOperator<?, ?>)) { return; } // get all broadcast inputs AbstractUdfOperator<?, ?> operator = ((AbstractUdfOperator<?, ?>) getOperator()); // create connections and add them for (Map.Entry<String, Operator<?>> input : operator.getBroadcastInputs().entrySet()) { OptimizerNode predecessor = operatorToNode.get(input.getValue()); DagConnection connection = new DagConnection(predecessor, this, ShipStrategyType.BROADCAST, defaultExchangeMode); addBroadcastConnection(input.getKey(), connection); predecessor.addOutgoingConnection(connection); } }
Example #3
Source File: ShuffleCompressionITCase.java From flink with Apache License 2.0 | 6 votes |
private static JobGraph createJobGraph( ScheduleMode scheduleMode, ResultPartitionType resultPartitionType, ExecutionMode executionMode) throws IOException { SlotSharingGroup slotSharingGroup = new SlotSharingGroup(); JobVertex source = new JobVertex("source"); source.setInvokableClass(LongValueSource.class); source.setParallelism(PARALLELISM); source.setSlotSharingGroup(slotSharingGroup); JobVertex sink = new JobVertex("sink"); sink.setInvokableClass(ResultVerifyingSink.class); sink.setParallelism(PARALLELISM); sink.setSlotSharingGroup(slotSharingGroup); sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType); JobGraph jobGraph = new JobGraph(source, sink); jobGraph.setScheduleMode(scheduleMode); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setExecutionMode(executionMode); jobGraph.setExecutionConfig(executionConfig); return jobGraph; }
Example #4
Source File: DataExchangeModeClosedBranchingTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testBatchForced() { // BATCH_FORCED should result in batching all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED, DataExchangeMode.BATCH, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.BATCH, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.BATCH, // to flatMap DataExchangeMode.BATCH, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.BATCH // to sink after coGroup ); }
Example #5
Source File: DataExchangeModeClosedBranchingTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.PIPELINED, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.PIPELINED, // to flatMap DataExchangeMode.PIPELINED, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.PIPELINED // to sink after coGroup ); }
Example #6
Source File: DataExchangeModeClosedBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testBatchForced() { // BATCH_FORCED should result in batching all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED, DataExchangeMode.BATCH, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.BATCH, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.BATCH, // to flatMap DataExchangeMode.BATCH, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.BATCH // to sink after coGroup ); }
Example #7
Source File: DataExchangeModeClosedBranchingTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.PIPELINED, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.PIPELINED, // to flatMap DataExchangeMode.PIPELINED, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.PIPELINED // to sink after coGroup ); }
Example #8
Source File: DataExchangeModeClosedBranchingTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testBatchForced() { // BATCH_FORCED should result in batching all the way verifyBranchingJoiningPlan(ExecutionMode.BATCH_FORCED, DataExchangeMode.BATCH, // to map DataExchangeMode.PIPELINED, // to combiner connections are pipelined DataExchangeMode.BATCH, // to reduce DataExchangeMode.BATCH, // to filter DataExchangeMode.BATCH, // to sink after reduce DataExchangeMode.BATCH, // to join (first input) DataExchangeMode.BATCH, // to join (second input) DataExchangeMode.PIPELINED, // combiner connections are pipelined DataExchangeMode.BATCH, // to other reducer DataExchangeMode.BATCH, // to flatMap DataExchangeMode.BATCH, // to sink after flatMap DataExchangeMode.BATCH, // to coGroup (first input) DataExchangeMode.BATCH, // to coGroup (second input) DataExchangeMode.BATCH // to sink after coGroup ); }
Example #9
Source File: GraphCreatingVisitor.java From flink with Apache License 2.0 | 5 votes |
private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism, ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) { if (closure == null){ con2node = new HashMap<Operator<?>, OptimizerNode>(); } else { con2node = closure; } this.sinks = new ArrayList<DataSinkNode>(2); this.defaultParallelism = defaultParallelism; this.parent = parent; this.defaultDataExchangeMode = dataExchangeMode; this.forceParallelism = forceParallelism; }
Example #10
Source File: DataExchangeModeForwardTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #11
Source File: SinkJoiner.java From flink with Apache License 2.0 | 5 votes |
public SinkJoiner(OptimizerNode input1, OptimizerNode input2) { super(new NoOpBinaryUdfOp<Nothing>(new NothingTypeInfo())); DagConnection conn1 = new DagConnection(input1, this, null, ExecutionMode.PIPELINED); DagConnection conn2 = new DagConnection(input2, this, null, ExecutionMode.PIPELINED); this.input1 = conn1; this.input2 = conn2; setParallelism(1); }
Example #12
Source File: DataExchangeModeForwardTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifySimpleForwardPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.BATCH, DataExchangeMode.PIPELINED); }
Example #13
Source File: DataExchangeModeForwardTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifySimpleForwardPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.BATCH, DataExchangeMode.PIPELINED); }
Example #14
Source File: GraphCreatingVisitor.java From flink with Apache License 2.0 | 5 votes |
private GraphCreatingVisitor(GraphCreatingVisitor parent, boolean forceParallelism, int defaultParallelism, ExecutionMode dataExchangeMode, HashMap<Operator<?>, OptimizerNode> closure) { if (closure == null){ con2node = new HashMap<Operator<?>, OptimizerNode>(); } else { con2node = closure; } this.sinks = new ArrayList<DataSinkNode>(2); this.defaultParallelism = defaultParallelism; this.parent = parent; this.defaultDataExchangeMode = dataExchangeMode; this.forceParallelism = forceParallelism; }
Example #15
Source File: PipelineBreakerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelineBreakerBroadcastedAllReduce() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setExecutionMode(ExecutionMode.PIPELINED); env.setParallelism(64); DataSet<Long> sourceWithMapper = env.generateSequence(1, 10).map(new IdentityMapper<Long>()); DataSet<Long> bcInput1 = sourceWithMapper .map(new IdentityMapper<Long>()) .reduce(new SelectOneReducer<Long>()); DataSet<Long> bcInput2 = env.generateSequence(1, 10); DataSet<Long> result = sourceWithMapper .map(new IdentityMapper<Long>()) .withBroadcastSet(bcInput1, "bc1") .withBroadcastSet(bcInput2, "bc2"); result.output(new DiscardingOutputFormat<Long>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(DataExchangeMode.BATCH, mapper.getInput().getDataExchangeMode()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #16
Source File: DataExchangeModeClosedBranchingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifyBranchingJoiningPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #17
Source File: FlinkPipelineOptionsTest.java From beam with Apache License 2.0 | 5 votes |
/** These defaults should only be changed with a very good reason. */ @Test public void testDefaults() { FlinkPipelineOptions options = PipelineOptionsFactory.as(FlinkPipelineOptions.class); assertThat(options.getParallelism(), is(-1)); assertThat(options.getMaxParallelism(), is(-1)); assertThat(options.getFlinkMaster(), is("[auto]")); assertThat(options.getFilesToStage(), is(nullValue())); assertThat(options.getLatencyTrackingInterval(), is(0L)); assertThat(options.getShutdownSourcesAfterIdleMs(), is(-1L)); assertThat(options.getObjectReuse(), is(false)); assertThat(options.getCheckpointingMode(), is(CheckpointingMode.EXACTLY_ONCE.name())); assertThat(options.getMinPauseBetweenCheckpoints(), is(-1L)); assertThat(options.getCheckpointingInterval(), is(-1L)); assertThat(options.getCheckpointTimeoutMillis(), is(-1L)); assertThat(options.getNumConcurrentCheckpoints(), is(1)); assertThat(options.getFailOnCheckpointingErrors(), is(true)); assertThat(options.getFinishBundleBeforeCheckpointing(), is(false)); assertThat(options.getNumberOfExecutionRetries(), is(-1)); assertThat(options.getExecutionRetryDelay(), is(-1L)); assertThat(options.getRetainExternalizedCheckpointsOnCancellation(), is(false)); assertThat(options.getStateBackendFactory(), is(nullValue())); assertThat(options.getMaxBundleSize(), is(1000L)); assertThat(options.getMaxBundleTimeMills(), is(1000L)); assertThat(options.getExecutionModeForBatch(), is(ExecutionMode.PIPELINED.name())); assertThat(options.getSavepointPath(), is(nullValue())); assertThat(options.getAllowNonRestoredState(), is(false)); assertThat(options.getDisableMetrics(), is(false)); }
Example #18
Source File: DataSinkNode.java From flink with Apache License 2.0 | 5 votes |
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) { Operator<?> children = getOperator().getInput(); final OptimizerNode pred; final DagConnection conn; pred = contractToNode.get(children); conn = new DagConnection(pred, this, defaultExchangeMode); // create the connection and add it this.input = conn; pred.addOutgoingConnection(conn); }
Example #19
Source File: DataExchangeModeOpenBranchingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #20
Source File: DataExchangeModeOpenBranchingTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifyBranchigPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #21
Source File: DataSinkNode.java From flink with Apache License 2.0 | 5 votes |
@Override public void setInput(Map<Operator<?>, OptimizerNode> contractToNode, ExecutionMode defaultExchangeMode) { Operator<?> children = getOperator().getInput(); final OptimizerNode pred; final DagConnection conn; pred = contractToNode.get(children); conn = new DagConnection(pred, this, defaultExchangeMode); // create the connection and add it this.input = conn; pred.addOutgoingConnection(conn); }
Example #22
Source File: ArchivedExecutionConfigBuilder.java From flink with Apache License 2.0 | 5 votes |
public ArchivedExecutionConfig build() { return new ArchivedExecutionConfig( executionMode != null ? executionMode : ExecutionMode.PIPELINED.name(), restartStrategyDescription != null ? restartStrategyDescription : "default", parallelism, objectReuseEnabled, globalJobParameters != null ? globalJobParameters : Collections.<String, String>emptyMap() ); }
Example #23
Source File: DataExchangeModeForwardTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #24
Source File: UnionClosedBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public UnionClosedBranchingTest( ExecutionMode executionMode, DataExchangeMode sourceToUnion, DataExchangeMode unionToJoin) { this.executionMode = executionMode; this.sourceToUnion = sourceToUnion; this.unionToJoin = unionToJoin; }
Example #25
Source File: PipelineBreakerTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testPipelineBreakerWithBroadcastVariable() { try { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().setExecutionMode(ExecutionMode.PIPELINED); env.setParallelism(64); DataSet<Long> source = env.generateSequence(1, 10).map(new IdentityMapper<Long>()); DataSet<Long> result = source.map(new IdentityMapper<Long>()) .map(new IdentityMapper<Long>()) .withBroadcastSet(source, "bc"); result.output(new DiscardingOutputFormat<Long>()); Plan p = env.createProgramPlan(); OptimizedPlan op = compileNoStats(p); SinkPlanNode sink = op.getDataSinks().iterator().next(); SingleInputPlanNode mapper = (SingleInputPlanNode) sink.getInput().getSource(); SingleInputPlanNode mapperInput = (SingleInputPlanNode) mapper.getInput().getSource(); assertEquals(TempMode.NONE, mapper.getInput().getTempMode()); assertEquals(TempMode.NONE, mapper.getBroadcastInputs().get(0).getTempMode()); assertEquals(DataExchangeMode.BATCH, mapperInput.getInput().getDataExchangeMode()); assertEquals(DataExchangeMode.BATCH, mapper.getBroadcastInputs().get(0).getDataExchangeMode()); } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
Example #26
Source File: DagConnection.java From flink with Apache License 2.0 | 5 votes |
/** * Gets the data exchange mode to use for this connection. * * @return The data exchange mode to use for this connection. */ public ExecutionMode getDataExchangeMode() { if (dataExchangeMode == null) { throw new IllegalStateException("This connection does not have the data exchange mode set"); } return dataExchangeMode; }
Example #27
Source File: DataExchangeModeForwardTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifySimpleForwardPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #28
Source File: DataExchangeModeOpenBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testBatchForced() { // BATCH_FORCED should result in batching all the way verifyBranchigPlan(ExecutionMode.BATCH_FORCED, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.BATCH); }
Example #29
Source File: DataExchangeModeOpenBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testBatch() { // BATCH should result in batching the shuffle all the way verifyBranchigPlan(ExecutionMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.BATCH, DataExchangeMode.BATCH, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }
Example #30
Source File: DataExchangeModeOpenBranchingTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testPipelinedForced() { // PIPELINED_FORCED should result in pipelining all the way verifyBranchigPlan(ExecutionMode.PIPELINED_FORCED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED, DataExchangeMode.PIPELINED); }