org.apache.flink.runtime.operators.testutils.DummyEnvironment Java Examples

The following examples show how to use org.apache.flink.runtime.operators.testutils.DummyEnvironment. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkStateInternalsTest.java    From beam with Apache License 2.0 6 votes vote down vote up
public static KeyedStateBackend<ByteBuffer> createStateBackend() throws Exception {
  MemoryStateBackend backend = new MemoryStateBackend();

  AbstractKeyedStateBackend<ByteBuffer> keyedStateBackend =
      backend.createKeyedStateBackend(
          new DummyEnvironment("test", 1, 0),
          new JobID(),
          "test_op",
          new GenericTypeInfo<>(ByteBuffer.class).createSerializer(new ExecutionConfig()),
          2,
          new KeyGroupRange(0, 1),
          new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
          TtlTimeProvider.DEFAULT,
          null,
          Collections.emptyList(),
          new CloseableRegistry());

  changeKey(keyedStateBackend);

  return keyedStateBackend;
}
 
Example #2
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSkipChannelStateForSavepoints() throws Exception {
	SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setUnalignedCheckpointEnabled(true)
		.setPrepareInputSnapshot((u1, u2) -> {
			fail("should not prepare input snapshot for savepoint");
			return null;
		}).build();

	coordinator.checkpointState(
		new CheckpointMetaData(0, 0),
		new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()),
		new CheckpointMetrics(),
		new OperatorChain<>(new NoOpStreamTask<>(new DummyEnvironment()), new NonRecordWriter<>()),
		() -> false);
}
 
Example #3
Source File: MultiStateKeyIteratorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static AbstractKeyedStateBackend<Integer> createKeyedStateBackend() {
	MockStateBackend backend = new MockStateBackend();

	return backend.createKeyedStateBackend(
		new DummyEnvironment(),
		new JobID(),
		"mock-backend",
		IntSerializer.INSTANCE,
		129,
		KeyGroupRange.of(0, 128),
		null,
		TtlTimeProvider.DEFAULT,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
 
Example #4
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void validateSupportForAsyncSnapshots(StateBackend backend) throws Exception {

		AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(
			new DummyEnvironment("Test", 1, 0),
			new JobID(),
			"testOperator",
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			null,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			new CloseableRegistry()
		);

		assertTrue(keyedStateBackend.supportsAsynchronousSnapshots());

		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
 
Example #5
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateSupportForAsyncSnapshots(StateBackend backend) throws Exception {

		AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(
			new DummyEnvironment("Test", 1, 0),
			new JobID(),
			"testOperator",
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			null,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			new CloseableRegistry()
		);

		assertTrue(keyedStateBackend.supportsAsynchronousSnapshots());

		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
 
Example #6
Source File: MultiStateKeyIteratorTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static AbstractKeyedStateBackend<Integer> createKeyedStateBackend() {
	MockStateBackend backend = new MockStateBackend();

	return backend.createKeyedStateBackend(
		new DummyEnvironment(),
		new JobID(),
		"mock-backend",
		IntSerializer.INSTANCE,
		129,
		KeyGroupRange.of(0, 128),
		null,
		TtlTimeProvider.DEFAULT,
		UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
 
Example #7
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void validateSupportForAsyncSnapshots(StateBackend backend) throws Exception {

		AbstractKeyedStateBackend<Integer> keyedStateBackend = backend.createKeyedStateBackend(
			new DummyEnvironment("Test", 1, 0),
			new JobID(),
			"testOperator",
			IntSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			null,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			new CloseableRegistry()
		);

		assertTrue(keyedStateBackend.supportsAsynchronousSnapshots());

		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
 
Example #8
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T> MockStreamTask setupSourceOperator(
		StreamSource<T, ?> operator,
		TimeCharacteristic timeChar,
		long watermarkInterval,
		final TimerService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setTimerService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
	return mockTask;
}
 
Example #9
Source File: StateBackendTestContext.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
void createAndRestoreKeyedStateBackend(int numberOfKeyGroups, KeyedStateHandle snapshot) {
	Collection<KeyedStateHandle> stateHandles;
	if (snapshot == null) {
		stateHandles = Collections.emptyList();
	} else {
		stateHandles = new ArrayList<>(1);
		stateHandles.add(snapshot);
	}
	Environment env = new DummyEnvironment();
	try {
		disposeKeyedStateBackend();
		keyedStateBackend = stateBackend.createKeyedStateBackend(
			env,
			new JobID(),
			"test",
			StringSerializer.INSTANCE,
			numberOfKeyGroups,
			new KeyGroupRange(0, numberOfKeyGroups - 1),
			env.getTaskKvStateRegistry(),
			timeProvider,
			new UnregisteredMetricsGroup(),
			stateHandles,
			new CloseableRegistry());
	} catch (Exception e) {
		throw new RuntimeException("unexpected", e);
	}
}
 
Example #10
Source File: TriggerTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
 
Example #11
Source File: KvStateServerHandlerTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
 
Example #12
Source File: StateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();
	Environment env = new DummyEnvironment();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);

	ExecutorService executorService = Executors.newScheduledThreadPool(1);
	try {
		long checkpointID = 0;
		List<Future> futureList = new ArrayList();
		for (int i = 0; i < 10; ++i) {
			ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE);
			ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
			((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
			backend.setCurrentKey(i);
			state.update(i);

			futureList.add(runSnapshotAsync(executorService,
				backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())));
		}

		for (Future future : futureList) {
			future.get(20, TimeUnit.SECONDS);
		}
	} catch (Exception e) {
		fail();
	} finally {
		backend.dispose();
		executorService.shutdown();
	}
}
 
Example #13
Source File: StreamingRuntimeContextTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createListPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<ListState<String>>() {

		@Override
		public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			ListStateDescriptor<String> descr =
					(ListStateDescriptor<String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(ListStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 
Example #14
Source File: SourceStreamTaskStoppingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testStop() {
	final StoppableSourceStreamTask<Object, StoppableSource> sourceTask =
			new StoppableSourceStreamTask<>(new DummyEnvironment("test", 1, 0));

	sourceTask.headOperator = new StoppableStreamSource<>(new StoppableSource());

	sourceTask.stop();

	assertTrue(stopped);
}
 
Example #15
Source File: StreamingRuntimeContextTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<MapState<Integer, String>>() {

		@Override
		public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			MapStateDescriptor<Integer, String> descr =
					(MapStateDescriptor<Integer, String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 
Example #16
Source File: StreamSourceOperatorWatermarksTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 
Example #17
Source File: TriggerTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
 
Example #18
Source File: SubtaskCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static SubtaskCheckpointCoordinator coordinator(boolean unalignedCheckpointEnabled, ChannelStateWriter channelStateWriter) throws IOException {
	return new SubtaskCheckpointCoordinatorImpl(
		new TestCheckpointStorageWorkerView(100),
		"test",
		StreamTaskActionExecutor.IMMEDIATE,
		new CloseableRegistry(),
		newDirectExecutorService(),
		new DummyEnvironment(),
		(message, unused) -> fail(message),
		(unused1, unused2) -> CompletableFuture.completedFuture(null),
		0,
		channelStateWriter
	);
}
 
Example #19
Source File: SourceStreamTaskStoppingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testStopBeforeInitialization() throws Exception {

	final StoppableSourceStreamTask<Object, StoppableFailingSource> sourceTask =
			new StoppableSourceStreamTask<>(new DummyEnvironment("test", 1, 0));
	sourceTask.stop();

	sourceTask.headOperator = new StoppableStreamSource<>(new StoppableFailingSource());
	sourceTask.run();
}
 
Example #20
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createListPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<ListState<String>>() {

		@Override
		public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			ListStateDescriptor<String> descr =
					(ListStateDescriptor<String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(ListStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 
Example #21
Source File: StreamTaskStateInitializerImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private StreamTaskStateInitializer streamTaskStateManager(
	StateBackend stateBackend,
	JobManagerTaskRestore jobManagerTaskRestore,
	boolean createTimerServiceManager) {

	JobID jobID = new JobID(42L, 43L);
	ExecutionAttemptID executionAttemptID = new ExecutionAttemptID(23L, 24L);
	TestCheckpointResponder checkpointResponderMock = new TestCheckpointResponder();

	TaskLocalStateStore taskLocalStateStore = new TestTaskLocalStateStore();

	TaskStateManager taskStateManager = TaskStateManagerImplTest.taskStateManager(
		jobID,
		executionAttemptID,
		checkpointResponderMock,
		jobManagerTaskRestore,
		taskLocalStateStore);

	DummyEnvironment dummyEnvironment = new DummyEnvironment("test-task", 1, 0);
	dummyEnvironment.setTaskStateManager(taskStateManager);

	if (createTimerServiceManager) {
		return new StreamTaskStateInitializerImpl(
			dummyEnvironment,
			stateBackend);
	} else {
		return new StreamTaskStateInitializerImpl(
			dummyEnvironment,
			stateBackend) {
			@Override
			protected <K> InternalTimeServiceManager<K> internalTimeServiceManager(
				AbstractKeyedStateBackend<K> keyedStatedBackend,
				KeyContext keyContext,
				ProcessingTimeService processingTimeService,
				Iterable<KeyGroupStatePartitionStreamProvider> rawKeyedStates) throws Exception {
				return null;
			}
		};
	}
}
 
Example #22
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createMapPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<MapState<Integer, String>>() {

		@Override
		public MapState<Integer, String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			MapStateDescriptor<Integer, String> descr =
					(MapStateDescriptor<Integer, String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(MapStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}
 
Example #23
Source File: FlinkBroadcastStateInternalsTest.java    From beam with Apache License 2.0 5 votes vote down vote up
@Override
protected StateInternals createStateInternals() {
  MemoryStateBackend backend = new MemoryStateBackend();
  try {
    OperatorStateBackend operatorStateBackend =
        backend.createOperatorStateBackend(
            new DummyEnvironment("test", 1, 0), "", Collections.emptyList(), null);
    return new FlinkBroadcastStateInternals<>(1, operatorStateBackend);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example #24
Source File: KvStateServerHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
 
Example #25
Source File: StateBackendTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckConcurrencyProblemWhenPerformingCheckpointAsync() throws Exception {

	CheckpointStreamFactory streamFactory = createStreamFactory();
	Environment env = new DummyEnvironment();
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE, env);

	ExecutorService executorService = Executors.newScheduledThreadPool(1);
	try {
		long checkpointID = 0;
		List<Future> futureList = new ArrayList();
		for (int i = 0; i < 10; ++i) {
			ValueStateDescriptor<Integer> kvId = new ValueStateDescriptor<>("id" + i, IntSerializer.INSTANCE);
			ValueState<Integer> state = backend.getOrCreateKeyedState(VoidNamespaceSerializer.INSTANCE, kvId);
			((InternalValueState) state).setCurrentNamespace(VoidNamespace.INSTANCE);
			backend.setCurrentKey(i);
			state.update(i);

			futureList.add(runSnapshotAsync(executorService,
				backend.snapshot(checkpointID++, System.currentTimeMillis(), streamFactory, CheckpointOptions.forCheckpointWithDefaultLocation())));
		}

		for (Future future : futureList) {
			future.get(20, TimeUnit.SECONDS);
		}
	} catch (Exception e) {
		fail();
	} finally {
		backend.dispose();
		executorService.shutdown();
	}
}
 
Example #26
Source File: KvStateServerHandlerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private AbstractKeyedStateBackend<Integer> createKeyedStateBackend(KvStateRegistry registry, int numKeyGroups, AbstractStateBackend abstractBackend, DummyEnvironment dummyEnv) throws java.io.IOException {
	return abstractBackend.createKeyedStateBackend(
		dummyEnv,
		dummyEnv.getJobID(),
		"test_op",
		IntSerializer.INSTANCE,
		numKeyGroups,
		new KeyGroupRange(0, 0),
		registry.createTaskRegistry(dummyEnv.getJobID(), dummyEnv.getJobVertexId()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
}
 
Example #27
Source File: TriggerTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
public TriggerTestHarness(
		Trigger<T, W> trigger,
		TypeSerializer<W> windowSerializer) throws Exception {
	this.trigger = trigger;
	this.windowSerializer = windowSerializer;

	// we only ever use one key, other tests make sure that windows work across different
	// keys
	DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0);
	MemoryStateBackend backend = new MemoryStateBackend();

	@SuppressWarnings("unchecked")
	HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(
		dummyEnv,
		new JobID(),
		"test_op",
		IntSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());
	this.stateBackend = stateBackend;

	this.stateBackend.setCurrentKey(KEY);

	this.internalTimerService = new TestInternalTimerService<>(new KeyContext() {
		@Override
		public void setCurrentKey(Object key) {
			// ignore
		}

		@Override
		public Object getCurrentKey() {
			return KEY;
		}
	});
}
 
Example #28
Source File: StateBackendTestContext.java    From flink with Apache License 2.0 5 votes vote down vote up
void createAndRestoreKeyedStateBackend(int numberOfKeyGroups, KeyedStateHandle snapshot) {
	Collection<KeyedStateHandle> stateHandles;
	if (snapshot == null) {
		stateHandles = Collections.emptyList();
	} else {
		stateHandles = new ArrayList<>(1);
		stateHandles.add(snapshot);
	}
	Environment env = new DummyEnvironment();
	try {
		disposeKeyedStateBackend();
		keyedStateBackend = stateBackend.createKeyedStateBackend(
			env,
			new JobID(),
			"test",
			StringSerializer.INSTANCE,
			numberOfKeyGroups,
			new KeyGroupRange(0, numberOfKeyGroups - 1),
			env.getTaskKvStateRegistry(),
			timeProvider,
			new UnregisteredMetricsGroup(),
			stateHandles,
			new CloseableRegistry());
	} catch (Exception e) {
		throw new RuntimeException("unexpected", e);
	}
}
 
Example #29
Source File: StreamSourceOperatorWatermarksTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static <T> void setupSourceOperator(StreamSource<T, ?> operator,
											TimeCharacteristic timeChar,
											long watermarkInterval,
											final ProcessingTimeService timeProvider) throws Exception {

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setAutoWatermarkInterval(watermarkInterval);

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateBackend(new MemoryStateBackend());

	cfg.setTimeCharacteristic(timeChar);
	cfg.setOperatorID(new OperatorID());

	Environment env = new DummyEnvironment("MockTwoInputTask", 1, 0);

	StreamStatusMaintainer streamStatusMaintainer = mock(StreamStatusMaintainer.class);
	when(streamStatusMaintainer.getStreamStatus()).thenReturn(StreamStatus.ACTIVE);

	MockStreamTask mockTask = new MockStreamTaskBuilder(env)
		.setConfig(cfg)
		.setExecutionConfig(executionConfig)
		.setStreamStatusMaintainer(streamStatusMaintainer)
		.setProcessingTimeService(timeProvider)
		.build();

	operator.setup(mockTask, cfg, (Output<StreamRecord<T>>) mock(Output.class));
}
 
Example #30
Source File: StreamingRuntimeContextTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static AbstractStreamOperator<?> createListPlainMockOp() throws Exception {

	AbstractStreamOperator<?> operatorMock = mock(AbstractStreamOperator.class);
	ExecutionConfig config = new ExecutionConfig();

	KeyedStateBackend keyedStateBackend = mock(KeyedStateBackend.class);

	DefaultKeyedStateStore keyedStateStore = new DefaultKeyedStateStore(keyedStateBackend, config);

	when(operatorMock.getExecutionConfig()).thenReturn(config);

	doAnswer(new Answer<ListState<String>>() {

		@Override
		public ListState<String> answer(InvocationOnMock invocationOnMock) throws Throwable {
			ListStateDescriptor<String> descr =
					(ListStateDescriptor<String>) invocationOnMock.getArguments()[2];

			AbstractKeyedStateBackend<Integer> backend = new MemoryStateBackend().createKeyedStateBackend(
				new DummyEnvironment("test_task", 1, 0),
				new JobID(),
				"test_op",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());
			backend.setCurrentKey(0);
			return backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, descr);
		}
	}).when(keyedStateBackend).getPartitionedState(Matchers.any(), any(TypeSerializer.class), any(ListStateDescriptor.class));

	when(operatorMock.getKeyedStateStore()).thenReturn(keyedStateStore);
	when(operatorMock.getOperatorID()).thenReturn(new OperatorID());
	return operatorMock;
}