org.apache.flink.runtime.execution.Environment Java Examples
The following examples show how to use
org.apache.flink.runtime.execution.Environment.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StreamSourceOperatorLatencyMetricsTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via * the configuration. */ @Test public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception { testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> { ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setLatencyTrackingInterval(latencyMarkInterval); Configuration tmConfig = new Configuration(); tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L); Environment env = MockEnvironment.builder() .setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig)) .build(); setupSourceOperator(operator, executionConfig, env, timeProvider); }); }
Example #2
Source File: StreamSourceOperatorLatencyMetricsTest.java From flink with Apache License 2.0 | 6 votes |
/** * Verifies that latency metrics can be enabled via the {@link ExecutionConfig} even if they are disabled via * the configuration. */ @Test public void testLatencyMarkEmissionEnabledOverrideViaExecutionConfig() throws Exception { testLatencyMarkEmission((int) (maxProcessingTime / latencyMarkInterval) + 1, (operator, timeProvider) -> { ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setLatencyTrackingInterval(latencyMarkInterval); Configuration tmConfig = new Configuration(); tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, 0L); Environment env = MockEnvironment.builder() .setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig)) .build(); setupSourceOperator(operator, executionConfig, env, timeProvider); }); }
Example #3
Source File: SubtaskCheckpointCoordinatorImpl.java From flink with Apache License 2.0 | 6 votes |
SubtaskCheckpointCoordinatorImpl( CheckpointStorageWorkerView checkpointStorage, String taskName, StreamTaskActionExecutor actionExecutor, CloseableRegistry closeableRegistry, ExecutorService executorService, Environment env, AsyncExceptionHandler asyncExceptionHandler, boolean unalignedCheckpointEnabled, BiFunctionWithException<ChannelStateWriter, Long, CompletableFuture<Void>, IOException> prepareInputSnapshot) throws IOException { this(checkpointStorage, taskName, actionExecutor, closeableRegistry, executorService, env, asyncExceptionHandler, unalignedCheckpointEnabled, prepareInputSnapshot, DEFAULT_MAX_RECORD_ABORTED_CHECKPOINTS); }
Example #4
Source File: StreamSourceOperatorLatencyMetricsTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Verifies that latency metrics can be disabled via the {@link ExecutionConfig} even if they are enabled via * the configuration. */ @Test public void testLatencyMarkEmissionDisabledOverrideViaExecutionConfig() throws Exception { testLatencyMarkEmission(0, (operator, timeProvider) -> { Configuration tmConfig = new Configuration(); tmConfig.setLong(MetricOptions.LATENCY_INTERVAL, latencyMarkInterval); Environment env = MockEnvironment.builder() .setTaskManagerRuntimeInfo(new TestingTaskManagerRuntimeInfo(tmConfig)) .build(); ExecutionConfig executionConfig = new ExecutionConfig(); executionConfig.setLatencyTrackingInterval(0); setupSourceOperator(operator, executionConfig, env, timeProvider); }); }
Example #5
Source File: StreamTask.java From flink with Apache License 2.0 | 6 votes |
private static <OUT> List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> createRecordWriters( StreamConfig configuration, Environment environment) { List<RecordWriter<SerializationDelegate<StreamRecord<OUT>>>> recordWriters = new ArrayList<>(); List<StreamEdge> outEdgesInOrder = configuration.getOutEdgesInOrder(environment.getUserClassLoader()); Map<Integer, StreamConfig> chainedConfigs = configuration.getTransitiveChainedTaskConfigsWithSelf(environment.getUserClassLoader()); for (int i = 0; i < outEdgesInOrder.size(); i++) { StreamEdge edge = outEdgesInOrder.get(i); recordWriters.add( createRecordWriter( edge, i, environment, environment.getTaskInfo().getTaskName(), chainedConfigs.get(edge.getSourceId()).getBufferTimeout())); } return recordWriters; }
Example #6
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private <K> AbstractKeyedStateBackend<K> restoreKeyedBackend( TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, List<KeyedStateHandle> state, Environment env) throws Exception { AbstractKeyedStateBackend<K> backend = getStateBackend().createKeyedStateBackend( env, new JobID(), "test_op", keySerializer, numberOfKeyGroups, keyGroupRange, env.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), state, new CloseableRegistry()); return backend; }
Example #7
Source File: KeyedStateInputFormat.java From flink with Apache License 2.0 | 6 votes |
private StreamOperatorStateContext getStreamOperatorStateContext(Environment environment) throws IOException { StreamTaskStateInitializer initializer = new StreamTaskStateInitializerImpl( environment, stateBackend); try { return initializer.streamOperatorStateContext( operatorState.getOperatorID(), operatorState.getOperatorID().toString(), new NeverFireProcessingTimeService(), operator, operator.getKeyType().createSerializer(environment.getExecutionConfig()), registry, getRuntimeContext().getMetricGroup()); } catch (Exception e) { throw new IOException("Failed to restore state backend", e); } }
Example #8
Source File: RocksDBStateBackend.java From flink with Apache License 2.0 | 6 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { //the default for RocksDB; eventually there can be a operator state backend based on RocksDB, too. final boolean asyncSnapshots = true; return new DefaultOperatorStateBackendBuilder( env.getUserClassLoader(), env.getExecutionConfig(), asyncSnapshots, stateHandles, cancelStreamRegistry).build(); }
Example #9
Source File: OneInputStreamTaskTestHarness.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Creates a test harness with the specified number of input gates and specified number * of channels per input gate and specified localRecoveryConfig. */ public OneInputStreamTaskTestHarness( Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory, int numInputGates, int numInputChannelsPerGate, TypeInformation<IN> inputType, TypeInformation<OUT> outputType, LocalRecoveryConfig localRecoveryConfig) { super(taskFactory, outputType, localRecoveryConfig); this.inputType = inputType; inputSerializer = inputType.createSerializer(executionConfig); this.numInputGates = numInputGates; this.numInputChannelsPerGate = numInputChannelsPerGate; }
Example #10
Source File: StateBackendTestBase.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
protected <K> AbstractKeyedStateBackend<K> createKeyedBackend( TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, Environment env) throws Exception { AbstractKeyedStateBackend<K> backend = getStateBackend().createKeyedStateBackend( env, new JobID(), "test_op", keySerializer, numberOfKeyGroups, keyGroupRange, env.getTaskKvStateRegistry(), TtlTimeProvider.DEFAULT, new UnregisteredMetricsGroup(), Collections.emptyList(), new CloseableRegistry()); return backend; }
Example #11
Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public StreamTaskStateInitializerImpl( Environment environment, StateBackend stateBackend, ProcessingTimeService processingTimeService) { this.environment = environment; this.taskStateManager = Preconditions.checkNotNull(environment.getTaskStateManager()); this.stateBackend = Preconditions.checkNotNull(stateBackend); this.processingTimeService = processingTimeService; }
Example #12
Source File: StateBackendITCase.java From flink with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws IOException { throw new SuccessException(); }
Example #13
Source File: StubStateBackend.java From flink with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return backend.createKeyedStateBackend( env, jobID, operatorIdentifier, keySerializer, numberOfKeyGroups, keyGroupRange, kvStateRegistry, this.ttlTimeProvider, metricGroup, stateHandles, cancelStreamRegistry); }
Example #14
Source File: StubStateBackend.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return backend.createKeyedStateBackend( env, jobID, operatorIdentifier, keySerializer, numberOfKeyGroups, keyGroupRange, kvStateRegistry, this.ttlTimeProvider, metricGroup, stateHandles, cancelStreamRegistry); }
Example #15
Source File: MemoryStateBackend.java From flink with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return new DefaultOperatorStateBackendBuilder( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots(), stateHandles, cancelStreamRegistry).build(); }
Example #16
Source File: MemoryStateBackend.java From flink with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return new DefaultOperatorStateBackendBuilder( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots(), stateHandles, cancelStreamRegistry).build(); }
Example #17
Source File: MockStateBackend.java From flink with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) { throw new UnsupportedOperationException(); }
Example #18
Source File: AbstractStreamOperatorTestHarness.java From flink with Apache License 2.0 | 5 votes |
protected StreamTaskStateInitializer createStreamTaskStateManager( Environment env, StateBackend stateBackend, TtlTimeProvider ttlTimeProvider) { return new StreamTaskStateInitializerImpl( env, stateBackend, ttlTimeProvider); }
Example #19
Source File: OneInputStreamTaskTestHarness.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Creates a test harness with the specified number of input gates and specified number * of channels per input gate and local recovery disabled. */ public OneInputStreamTaskTestHarness( Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory, int numInputGates, int numInputChannelsPerGate, TypeInformation<IN> inputType, TypeInformation<OUT> outputType) { this(taskFactory, numInputGates, numInputChannelsPerGate, inputType, outputType, TestLocalRecoveryConfig.disabled()); }
Example #20
Source File: RocksDBStateBackendConfigTest.java From flink with Apache License 2.0 | 5 votes |
/** * This test checks the behavior for basic setting of local DB directories. */ @Test public void testSetDbPath() throws Exception { final RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(tempFolder.newFolder().toURI().toString()); final String testDir1 = tempFolder.newFolder().getAbsolutePath(); final String testDir2 = tempFolder.newFolder().getAbsolutePath(); assertNull(rocksDbBackend.getDbStoragePaths()); rocksDbBackend.setDbStoragePath(testDir1); assertArrayEquals(new String[] { testDir1 }, rocksDbBackend.getDbStoragePaths()); rocksDbBackend.setDbStoragePath(null); assertNull(rocksDbBackend.getDbStoragePaths()); rocksDbBackend.setDbStoragePaths(testDir1, testDir2); assertArrayEquals(new String[] { testDir1, testDir2 }, rocksDbBackend.getDbStoragePaths()); final Environment env = getMockEnvironment(tempFolder.newFolder()); final RocksDBKeyedStateBackend<Integer> keyedBackend = createKeyedStateBackend(rocksDbBackend, env); try { File instanceBasePath = keyedBackend.getInstanceBasePath(); assertThat(instanceBasePath.getAbsolutePath(), anyOf(startsWith(testDir1), startsWith(testDir2))); //noinspection NullArgumentToVariableArgMethod rocksDbBackend.setDbStoragePaths(null); assertNull(rocksDbBackend.getDbStoragePaths()); } finally { IOUtils.closeQuietly(keyedBackend); keyedBackend.dispose(); } }
Example #21
Source File: SometimesInstantiationErrorSender.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public SometimesInstantiationErrorSender(Environment environment) { super(environment); if (failingSenders.contains(this.getIndexInSubtaskGroup())) { throw new RuntimeException("Test exception in constructor"); } }
Example #22
Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0 | 5 votes |
@VisibleForTesting public StreamTaskStateInitializerImpl( Environment environment, StateBackend stateBackend, TtlTimeProvider ttlTimeProvider) { this.environment = environment; this.taskStateManager = Preconditions.checkNotNull(environment.getTaskStateManager()); this.stateBackend = Preconditions.checkNotNull(stateBackend); this.ttlTimeProvider = ttlTimeProvider; }
Example #23
Source File: StateBackendMigrationTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private <K> AbstractKeyedStateBackend<K> createKeyedBackend(TypeSerializer<K> keySerializer, Environment env) throws Exception { return createKeyedBackend( keySerializer, 10, new KeyGroupRange(0, 9), env); }
Example #24
Source File: FsStateBackend.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws BackendBuildingException { TaskStateManager taskStateManager = env.getTaskStateManager(); LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig(); HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); return new HeapKeyedStateBackendBuilder<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, env.getExecutionConfig(), ttlTimeProvider, stateHandles, AbstractStateBackend.getCompressionDecorator(env.getExecutionConfig()), localRecoveryConfig, priorityQueueSetFactory, isUsingAsynchronousSnapshots(), cancelStreamRegistry).build(); }
Example #25
Source File: StreamTaskTerminationTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) { return null; }
Example #26
Source File: MemoryStateBackend.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return new DefaultOperatorStateBackendBuilder( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots(), stateHandles, cancelStreamRegistry).build(); }
Example #27
Source File: StateBackendTestBase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception { CheckpointStreamFactory streamFactory = createStreamFactory(); Environment env = new DummyEnvironment(); AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env); ExecutorService executorService = Executors.newScheduledThreadPool(1); try { long checkpointID = 0; List<Future> futureList = new ArrayList(); for (int i = 0; i < 10; ++i) { ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE); ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId); ((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE); backend.setCurrentKey(i); state.update(i); futureList.add(runSnapshotAsync(executorService, backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation()))); } for (Future future : futureList) { future.get(20, TimeUnit.SECONDS); } } catch (Exception e) { fail(); } finally { backend.dispose(); executorService.shutdown(); } }
Example #28
Source File: StubStateBackend.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws Exception { return backend.createOperatorStateBackend(env, operatorIdentifier, stateHandles, cancelStreamRegistry); }
Example #29
Source File: FsStateBackend.java From flink with Apache License 2.0 | 5 votes |
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier, @Nonnull Collection<OperatorStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) throws BackendBuildingException { return new DefaultOperatorStateBackendBuilder( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots(), stateHandles, cancelStreamRegistry).build(); }
Example #30
Source File: StreamTaskTerminationTest.java From flink with Apache License 2.0 | 5 votes |
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup, @Nonnull Collection<KeyedStateHandle> stateHandles, CloseableRegistry cancelStreamRegistry) { return null; }