Java Code Examples for org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#setStateBackend()

The following examples show how to use org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness#setStateBackend() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
 
Example 2
Source Project: Flink-CEPplus   File: CEPOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
 
Example 3
Source Project: flink   File: PojoSerializerUpgradeTest.java    License: Apache License 2.0 4 votes vote down vote up
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
 
Example 4
Source Project: flink   File: CEPOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}
 
Example 5
Source Project: flink   File: PojoSerializerUpgradeTest.java    License: Apache License 2.0 4 votes vote down vote up
private OperatorSubtaskState runOperator(
		Configuration taskConfiguration,
		ExecutionConfig executionConfig,
		OneInputStreamOperator<Long, Long> operator,
		KeySelector<Long, Long> keySelector,
		boolean isKeyedState,
		StateBackend stateBackend,
		ClassLoader classLoader,
		OperatorSubtaskState operatorSubtaskState,
		Iterable<Long> input) throws Exception {

	try (final MockEnvironment environment =
			new MockEnvironmentBuilder()
				.setTaskName("test task")
				.setManagedMemorySize(32 * 1024)
				.setInputSplitProvider(new MockInputSplitProvider())
				.setBufferSize(256)
				.setTaskConfiguration(taskConfiguration)
				.setExecutionConfig(executionConfig)
				.setMaxParallelism(16)
				.setUserCodeClassLoader(classLoader)
				.build()) {

		OneInputStreamOperatorTestHarness<Long, Long> harness = null;
		try {
			if (isKeyedState) {
				harness = new KeyedOneInputStreamOperatorTestHarness<>(
					operator,
					keySelector,
					BasicTypeInfo.LONG_TYPE_INFO,
					environment);
			} else {
				harness = new OneInputStreamOperatorTestHarness<>(operator, LongSerializer.INSTANCE, environment);
			}

			harness.setStateBackend(stateBackend);

			harness.setup();
			harness.initializeState(operatorSubtaskState);
			harness.open();

			long timestamp = 0L;

			for (Long value : input) {
				harness.processElement(value, timestamp++);
			}

			long checkpointId = 1L;
			long checkpointTimestamp = timestamp + 1L;

			return harness.snapshot(checkpointId, checkpointTimestamp);
		} finally {
			IOUtils.closeQuietly(harness);
		}
	}
}
 
Example 6
Source Project: flink   File: CEPOperatorTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testKeyedCEPOperatorNFAUpdateTimesWithRocksDB() throws Exception {

	String rocksDbPath = tempFolder.newFolder().getAbsolutePath();
	RocksDBStateBackend rocksDBStateBackend = new RocksDBStateBackend(new MemoryStateBackend());
	rocksDBStateBackend.setDbStoragePath(rocksDbPath);

	CepOperator<Event, Integer, Map<String, List<Event>>> operator = CepOperatorTestUtilities.getKeyedCepOpearator(
		true,
		new SimpleNFAFactory());
	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = CepOperatorTestUtilities.getCepTestHarness(
		operator);

	try {
		harness.setStateBackend(rocksDBStateBackend);

		harness.open();

		final ValueState nfaOperatorState = (ValueState) Whitebox.<ValueState>getInternalState(operator, "computationStates");
		final ValueState nfaOperatorStateSpy = Mockito.spy(nfaOperatorState);
		Whitebox.setInternalState(operator, "computationStates", nfaOperatorStateSpy);

		Event startEvent = new Event(42, "c", 1.0);
		SubEvent middleEvent = new SubEvent(42, "a", 1.0, 10.0);
		Event endEvent = new Event(42, "b", 1.0);

		harness.processElement(new StreamRecord<>(startEvent, 1L));
		harness.processElement(new StreamRecord<>(new Event(42, "d", 1.0), 4L));
		harness.processElement(new StreamRecord<Event>(middleEvent, 4L));
		harness.processElement(new StreamRecord<>(endEvent, 4L));

		// verify the number of invocations NFA is updated
		Mockito.verify(nfaOperatorStateSpy, Mockito.times(3)).update(Mockito.any());

		// get and verify the output
		Queue<Object> result = harness.getOutput();

		assertEquals(1, result.size());

		verifyPattern(result.poll(), startEvent, middleEvent, endEvent);
	} finally {
		harness.close();
	}
}