org.apache.flink.runtime.state.TestLocalRecoveryConfig Java Examples

The following examples show how to use org.apache.flink.runtime.state.TestLocalRecoveryConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
 
Example #2
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
 
Example #3
Source File: KvStateRequestSerializerTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
 
Example #4
Source File: RocksDBTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static <K> RocksDBKeyedStateBackendBuilder<K> builderForTestDefaults(
		File instanceBasePath,
		TypeSerializer<K> keySerializer) {

	final RocksDBResourceContainer optionsContainer = new RocksDBResourceContainer();

	return new RocksDBKeyedStateBackendBuilder<>(
		"no-op",
		ClassLoader.getSystemClassLoader(),
		instanceBasePath,
		optionsContainer,
		stateName -> optionsContainer.getColumnOptions(),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		keySerializer,
		2,
		new KeyGroupRange(0, 1),
		new ExecutionConfig(),
		TestLocalRecoveryConfig.disabled(),
		RocksDBStateBackend.PriorityQueueStateType.HEAP,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		UncompressedStreamCompressionDecorator.INSTANCE,
		new CloseableRegistry());
}
 
Example #5
Source File: HeapStateBackendTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
 
Example #6
Source File: KvStateRequestSerializerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private HeapKeyedStateBackend<Long> getLongHeapKeyedStateBackend(final long key) throws BackendBuildingException {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0);
	ExecutionConfig executionConfig = new ExecutionConfig();
	// objects for heap state list serialisation
	final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new HeapKeyedStateBackendBuilder<>(
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			ClassLoader.getSystemClassLoader(),
			keyGroupRange.getNumberOfKeyGroups(),
			keyGroupRange,
			executionConfig,
			TtlTimeProvider.DEFAULT,
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			TestLocalRecoveryConfig.disabled(),
			new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128),
			async,
			new CloseableRegistry()).build();
	longHeapKeyedStateBackend.setCurrentKey(key);
	return longHeapKeyedStateBackend;
}
 
Example #7
Source File: HeapStateBackendTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public <K> HeapKeyedStateBackend<K> createKeyedBackend(
	TypeSerializer<K> keySerializer,
	Collection<KeyedStateHandle> stateHandles) throws Exception {
	final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 15);
	final int numKeyGroups = keyGroupRange.getNumberOfKeyGroups();
	ExecutionConfig executionConfig = new ExecutionConfig();

	return new HeapKeyedStateBackendBuilder<>(
		mock(TaskKvStateRegistry.class),
		keySerializer,
		HeapStateBackendTestBase.class.getClassLoader(),
		numKeyGroups,
		keyGroupRange,
		executionConfig,
		TtlTimeProvider.DEFAULT,
		stateHandles,
		AbstractStateBackend.getCompressionDecorator(executionConfig),
		TestLocalRecoveryConfig.disabled(),
		new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128),
		async,
		new CloseableRegistry()).build();
}
 
Example #8
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with one input gate that has one input channel.
 */
public OneInputStreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, 1, 1, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #9
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with one input gate that has one input channel.
 */
public OneInputStreamTaskTestHarness(
	FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, 1, 1, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #10
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and local recovery disabled.
 */
public OneInputStreamTaskTestHarness(
	FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
	int numInputGates,
	int numInputChannelsPerGate,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, numInputGates, numInputChannelsPerGate, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #11
Source File: RocksDBTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <K> RocksDBKeyedStateBackendBuilder<K> builderForTestDB(
		File instanceBasePath,
		TypeSerializer<K> keySerializer,
		RocksDB db,
		ColumnFamilyHandle defaultCFHandle,
		ColumnFamilyOptions columnFamilyOptions) {

	final RocksDBResourceContainer optionsContainer = new RocksDBResourceContainer();

	return new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			instanceBasePath,
			optionsContainer,
			stateName -> columnFamilyOptions,
			new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
			keySerializer,
			2,
			new KeyGroupRange(0, 1),
			new ExecutionConfig(),
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			UncompressedStreamCompressionDecorator.INSTANCE,
			db,
			defaultCFHandle,
			new CloseableRegistry());
}
 
Example #12
Source File: KVStateRequestSerializerRocksDBTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 
Example #13
Source File: OneInputStreamTaskTestHarness.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and local recovery disabled.
 */
public OneInputStreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	int numInputGates,
	int numInputChannelsPerGate,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, numInputGates, numInputChannelsPerGate, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #14
Source File: KVStateRequestSerializerRocksDBTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests list serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testListSerialization()
 * KvStateRequestSerializerTest#testListSerialization() using the heap state back-end
 * test
 */
@Test
public void testListSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState(VoidNamespaceSerializer.INSTANCE,
			new ListStateDescriptor<>("test", LongSerializer.INSTANCE));

	KvStateRequestSerializerTest.testListSerialization(key, listState);
	longHeapKeyedStateBackend.dispose();
}
 
Example #15
Source File: OneInputStreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with one input gate that has one input channel.
 */
public OneInputStreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, 1, 1, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #16
Source File: OneInputStreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a test harness with the specified number of input gates and specified number
 * of channels per input gate and local recovery disabled.
 */
public OneInputStreamTaskTestHarness(
	Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
	int numInputGates,
	int numInputChannelsPerGate,
	TypeInformation<IN> inputType,
	TypeInformation<OUT> outputType) {
	this(taskFactory, numInputGates, numInputChannelsPerGate, inputType, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #17
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 4 votes vote down vote up
public StreamTaskTestHarness(
		Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
		TypeInformation<OUT> outputType) {
	this(taskFactory, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #18
Source File: JvmExitOnFatalErrorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #19
Source File: KVStateRequestSerializerRocksDBTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}
 
Example #20
Source File: KVStateRequestSerializerRocksDBTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests map serialization and deserialization match.
 *
 * @see KvStateRequestSerializerTest#testMapSerialization()
 * KvStateRequestSerializerTest#testMapSerialization() using the heap state back-end
 * test
 */
@Test
public void testMapSerialization() throws Exception {
	final long key = 0L;

	// objects for RocksDB state list serialisation
	DBOptions dbOptions = PredefinedOptions.DEFAULT.createDBOptions();
	dbOptions.setCreateIfMissing(true);
	ExecutionConfig executionConfig = new ExecutionConfig();
	final RocksDBKeyedStateBackend<Long> longHeapKeyedStateBackend =
		new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			temporaryFolder.getRoot(),
			dbOptions,
			stateName -> PredefinedOptions.DEFAULT.createColumnOptions(),
			mock(TaskKvStateRegistry.class),
			LongSerializer.INSTANCE,
			1,
			new KeyGroupRange(0, 0),
			executionConfig,
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			AbstractStateBackend.getCompressionDecorator(executionConfig),
			new CloseableRegistry()
		).build();
	longHeapKeyedStateBackend.setCurrentKey(key);

	final InternalMapState<Long, VoidNamespace, Long, String> mapState =
			(InternalMapState<Long, VoidNamespace, Long, String>)
					longHeapKeyedStateBackend.getPartitionedState(
							VoidNamespace.INSTANCE,
							VoidNamespaceSerializer.INSTANCE,
							new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE));

	KvStateRequestSerializerTest.testMapSerialization(key, mapState);
	longHeapKeyedStateBackend.dispose();
}
 
Example #21
Source File: JvmExitOnFatalErrorTest.java    From flink with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
				final IOManager ioManager = new IOManagerAsync();

				final ShuffleEnvironment<?, ?> shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();

				final Configuration copiedConf = new Configuration(taskManagerConfig);
				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration
					.fromConfiguration(
						taskManagerConfig,
						TaskExecutorResourceUtils.resourceSpecFromConfigForLocalExecution(copiedConf),
						InetAddress.getLoopbackAddress().getHostAddress());

				final Executor executor = Executors.newCachedThreadPool();

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						shuffleEnvironment,
						new KvStateService(new KvStateRegistry(), null, null),
						new BroadcastVariableManager(),
						new TaskEventDispatcher(),
						ExternalResourceInfoProvider.NO_EXTERNAL_RESOURCES,
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						NoOpCheckpointResponder.INSTANCE,
						new NoOpTaskOperatorEventGateway(),
						new TestGlobalAggregateManager(),
						TestingClassLoaderLease.newBuilder().build(),
						new FileCache(tmInfo.getTmpDirectories(), VoidPermanentBlobService.INSTANCE),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}
 
Example #22
Source File: StreamTaskTestHarness.java    From flink with Apache License 2.0 4 votes vote down vote up
public StreamTaskTestHarness(
		FunctionWithException<Environment, ? extends StreamTask<OUT, ?>, Exception> taskFactory,
		TypeInformation<OUT> outputType) {
	this(taskFactory, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #23
Source File: StreamTaskTestHarness.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public StreamTaskTestHarness(
		Function<Environment, ? extends StreamTask<OUT, ?>> taskFactory,
		TypeInformation<OUT> outputType) {
	this(taskFactory, outputType, TestLocalRecoveryConfig.disabled());
}
 
Example #24
Source File: JvmExitOnFatalErrorTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {

			System.err.println("creating task");

			// we suppress process exits via errors here to not
			// have a test that exits accidentally due to a programming error
			try {
				final Configuration taskManagerConfig = new Configuration();
				taskManagerConfig.setBoolean(TaskManagerOptions.KILL_ON_OUT_OF_MEMORY, true);

				final JobID jid = new JobID();
				final AllocationID allocationID = new AllocationID();
				final JobVertexID jobVertexId = new JobVertexID();
				final ExecutionAttemptID executionAttemptID = new ExecutionAttemptID();
				final AllocationID slotAllocationId = new AllocationID();

				final SerializedValue<ExecutionConfig> execConfig = new SerializedValue<>(new ExecutionConfig());

				final JobInformation jobInformation = new JobInformation(
						jid, "Test Job", execConfig, new Configuration(),
						Collections.emptyList(), Collections.emptyList());

				final TaskInformation taskInformation = new TaskInformation(
						jobVertexId, "Test Task", 1, 1, OomInvokable.class.getName(), new Configuration());

				final MemoryManager memoryManager = new MemoryManager(1024 * 1024, 1);
				final IOManager ioManager = new IOManagerAsync();

				final NetworkEnvironment networkEnvironment = mock(NetworkEnvironment.class);
				when(networkEnvironment.createKvStateTaskRegistry(jid, jobVertexId)).thenReturn(mock(TaskKvStateRegistry.class));
				TaskEventDispatcher taskEventDispatcher = mock(TaskEventDispatcher.class);
				when(networkEnvironment.getTaskEventDispatcher()).thenReturn(taskEventDispatcher);

				final TaskManagerRuntimeInfo tmInfo = TaskManagerConfiguration.fromConfiguration(taskManagerConfig);

				final Executor executor = Executors.newCachedThreadPool();

				BlobCacheService blobService =
					new BlobCacheService(mock(PermanentBlobCache.class), mock(TransientBlobCache.class));

				final TaskLocalStateStore localStateStore =
					new TaskLocalStateStoreImpl(
						jid,
						allocationID,
						jobVertexId,
						0,
						TestLocalRecoveryConfig.disabled(),
						executor);

				final TaskStateManager slotStateManager =
					new TaskStateManagerImpl(
						jid,
						executionAttemptID,
						localStateStore,
						null,
						mock(CheckpointResponder.class));

				Task task = new Task(
						jobInformation,
						taskInformation,
						executionAttemptID,
						slotAllocationId,
						0,       // subtaskIndex
						0,       // attemptNumber
						Collections.<ResultPartitionDeploymentDescriptor>emptyList(),
						Collections.<InputGateDeploymentDescriptor>emptyList(),
						0,       // targetSlotNumber
						memoryManager,
						ioManager,
						networkEnvironment,
						new BroadcastVariableManager(),
						slotStateManager,
						new NoOpTaskManagerActions(),
						new NoOpInputSplitProvider(),
						new NoOpCheckpointResponder(),
						new TestGlobalAggregateManager(),
						blobService,
						new BlobLibraryCacheManager(
							blobService.getPermanentBlobService(),
							FlinkUserCodeClassLoaders.ResolveOrder.CHILD_FIRST,
							new String[0]),
						new FileCache(tmInfo.getTmpDirectories(), blobService.getPermanentBlobService()),
						tmInfo,
						UnregisteredMetricGroups.createUnregisteredTaskMetricGroup(),
						new NoOpResultPartitionConsumableNotifier(),
						new NoOpPartitionProducerStateChecker(),
						executor);

				System.err.println("starting task thread");

				task.startTaskThread();
			}
			catch (Throwable t) {
				System.err.println("ERROR STARTING TASK");
				t.printStackTrace();
			}

			System.err.println("parking the main thread");
			CommonTestUtils.blockForeverNonInterruptibly();
		}