Java Code Examples for org.apache.flink.runtime.state.AbstractKeyedStateBackend#dispose()

The following examples show how to use org.apache.flink.runtime.state.AbstractKeyedStateBackend#dispose() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}

Example 2

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}

Example 3

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

5 votes

@Test
public void testDisposeDeletesAllDirectories() throws Exception {
	AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
	Collection<File> allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());
	try {
		ValueStateDescriptor<String> kvId =
			new ValueStateDescriptor<>("id", String.class, null);

		kvId.initializeSerializerUnlessSet(new ExecutionConfig());

		ValueState<String> state =
			backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

		backend.setCurrentKey(1);
		state.update("Hello");

		// more than just the root directory
		assertTrue(allFilesInDbDir.size() > 1);
	} finally {
		IOUtils.closeQuietly(backend);
		backend.dispose();
	}
	allFilesInDbDir =
		FileUtils.listFilesAndDirs(new File(dbPath), new AcceptAllFilter(), new AcceptAllFilter());

	// just the root directory left
	assertEquals(1, allFilesInDbDir.size());
}

Example 4

Source File: RocksDBStateBackendConfigTest.java From flink with Apache License 2.0

5 votes

@Test
public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
	final File targetDir1 = tempFolder.newFolder();
	final File targetDir2 = tempFolder.newFolder();
	Assume.assumeTrue("Cannot mark directory non-writable", targetDir1.setWritable(false, false));

	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	try (MockEnvironment env = getMockEnvironment(tempFolder.newFolder())) {
		rocksDbBackend.setDbStoragePaths(targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());

		try {
			AbstractKeyedStateBackend<Integer> keyedStateBackend = rocksDbBackend.createKeyedStateBackend(
				env,
				env.getJobID(),
				"foobar",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());

			IOUtils.closeQuietly(keyedStateBackend);
			keyedStateBackend.dispose();
		}
		catch (Exception e) {
			e.printStackTrace();
			fail("Backend initialization failed even though some paths were available");
		}
	} finally {
		//noinspection ResultOfMethodCallIgnored
		targetDir1.setWritable(true, false);
	}
}

Example 5

Source File: RocksDBStateBackendTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}

Example 6

Source File: RocksDBStateBackendConfigTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
	File targetDir1 = tempFolder.newFolder();
	File targetDir2 = tempFolder.newFolder();

	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	try {

		if (!targetDir1.setWritable(false, false)) {
			System.err.println("Cannot execute 'testContinueOnSomeDbDirectoriesMissing' because cannot mark directory non-writable");
			return;
		}

		rocksDbBackend.setDbStoragePaths(targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());

		try {
			Environment env = getMockEnvironment(tempFolder.newFolder());
			AbstractKeyedStateBackend<Integer> keyedStateBackend = rocksDbBackend.createKeyedStateBackend(
				env,
				env.getJobID(),
				"foobar",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());

			IOUtils.closeQuietly(keyedStateBackend);
			keyedStateBackend.dispose();
		}
		catch (Exception e) {
			e.printStackTrace();
			fail("Backend initialization failed even though some paths were available");
		}
	} finally {
		//noinspection ResultOfMethodCallIgnored
		targetDir1.setWritable(true, false);
	}
}

Example 7

Source File: RocksDBAsyncSnapshotTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	Environment env = new DummyEnvironment("test task", 1, 0);

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
}

Example 8

Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public StreamOperatorStateContext streamOperatorStateContext(
	@Nonnull OperatorID operatorID,
	@Nonnull String operatorClassName,
	@Nonnull KeyContext keyContext,
	@Nullable TypeSerializer<?> keySerializer,
	@Nonnull CloseableRegistry streamTaskCloseableRegistry,
	@Nonnull MetricGroup metricGroup) throws Exception {

	TaskInfo taskInfo = environment.getTaskInfo();
	OperatorSubtaskDescriptionText operatorSubtaskDescription =
		new OperatorSubtaskDescriptionText(
			operatorID,
			operatorClassName,
			taskInfo.getIndexOfThisSubtask(),
			taskInfo.getNumberOfParallelSubtasks());

	final String operatorIdentifierText = operatorSubtaskDescription.toString();

	final PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates =
		taskStateManager.prioritizedOperatorState(operatorID);

	AbstractKeyedStateBackend<?> keyedStatedBackend = null;
	OperatorStateBackend operatorStateBackend = null;
	CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs = null;
	CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs = null;
	InternalTimeServiceManager<?> timeServiceManager;

	try {

		// -------------- Keyed State Backend --------------
		keyedStatedBackend = keyedStatedBackend(
			keySerializer,
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry,
			metricGroup);

		// -------------- Operator State Backend --------------
		operatorStateBackend = operatorStateBackend(
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry);

		// -------------- Raw State Streams --------------
		rawKeyedStateInputs = rawKeyedStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs);

		rawOperatorStateInputs = rawOperatorStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs);

		// -------------- Internal Timer Service Manager --------------
		timeServiceManager = internalTimeServiceManager(keyedStatedBackend, keyContext, rawKeyedStateInputs);

		// -------------- Preparing return value --------------

		return new StreamOperatorStateContextImpl(
			prioritizedOperatorSubtaskStates.isRestored(),
			operatorStateBackend,
			keyedStatedBackend,
			timeServiceManager,
			rawOperatorStateInputs,
			rawKeyedStateInputs);
	} catch (Exception ex) {

		// cleanup if something went wrong before results got published.
		if (keyedStatedBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(keyedStatedBackend)) {
				IOUtils.closeQuietly(keyedStatedBackend);
			}
			// release resource (e.g native resource)
			keyedStatedBackend.dispose();
		}

		if (operatorStateBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(operatorStateBackend)) {
				IOUtils.closeQuietly(operatorStateBackend);
			}
			operatorStateBackend.dispose();
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawKeyedStateInputs)) {
			IOUtils.closeQuietly(rawKeyedStateInputs);
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawOperatorStateInputs)) {
			IOUtils.closeQuietly(rawOperatorStateInputs);
		}

		throw new Exception("Exception while creating StreamOperatorStateContext.", ex);
	}
}

Example 9

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}

Example 10

Source File: RocksDBStateBackendConfigTest.java From flink with Apache License 2.0

4 votes

@Test
public void testContinueOnSomeDbDirectoriesMissing() throws Exception {
	File targetDir1 = tempFolder.newFolder();
	File targetDir2 = tempFolder.newFolder();

	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend rocksDbBackend = new RocksDBStateBackend(checkpointPath);

	try {

		if (!targetDir1.setWritable(false, false)) {
			System.err.println("Cannot execute 'testContinueOnSomeDbDirectoriesMissing' because cannot mark directory non-writable");
			return;
		}

		rocksDbBackend.setDbStoragePaths(targetDir1.getAbsolutePath(), targetDir2.getAbsolutePath());

		try {
			Environment env = getMockEnvironment(tempFolder.newFolder());
			AbstractKeyedStateBackend<Integer> keyedStateBackend = rocksDbBackend.createKeyedStateBackend(
				env,
				env.getJobID(),
				"foobar",
				IntSerializer.INSTANCE,
				1,
				new KeyGroupRange(0, 0),
				new KvStateRegistry().createTaskRegistry(env.getJobID(), new JobVertexID()),
				TtlTimeProvider.DEFAULT,
				new UnregisteredMetricsGroup(),
				Collections.emptyList(),
				new CloseableRegistry());

			IOUtils.closeQuietly(keyedStateBackend);
			keyedStateBackend.dispose();
		}
		catch (Exception e) {
			e.printStackTrace();
			fail("Backend initialization failed even though some paths were available");
		}
	} finally {
		//noinspection ResultOfMethodCallIgnored
		targetDir1.setWritable(true, false);
	}
}

Example 11

Source File: RocksDBAsyncSnapshotTest.java From flink with Apache License 2.0

4 votes

/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	Environment env = new DummyEnvironment("test task", 1, 0);

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
	}
}

Example 12

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

4 votes

@Override
public StreamOperatorStateContext streamOperatorStateContext(
	@Nonnull OperatorID operatorID,
	@Nonnull String operatorClassName,
	@Nonnull KeyContext keyContext,
	@Nullable TypeSerializer<?> keySerializer,
	@Nonnull CloseableRegistry streamTaskCloseableRegistry,
	@Nonnull MetricGroup metricGroup) throws Exception {

	TaskInfo taskInfo = environment.getTaskInfo();
	OperatorSubtaskDescriptionText operatorSubtaskDescription =
		new OperatorSubtaskDescriptionText(
			operatorID,
			operatorClassName,
			taskInfo.getIndexOfThisSubtask(),
			taskInfo.getNumberOfParallelSubtasks());

	final String operatorIdentifierText = operatorSubtaskDescription.toString();

	final PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates =
		taskStateManager.prioritizedOperatorState(operatorID);

	AbstractKeyedStateBackend<?> keyedStatedBackend = null;
	OperatorStateBackend operatorStateBackend = null;
	CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs = null;
	CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs = null;
	InternalTimeServiceManager<?> timeServiceManager;

	try {

		// -------------- Keyed State Backend --------------
		keyedStatedBackend = keyedStatedBackend(
			keySerializer,
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry,
			metricGroup);

		// -------------- Operator State Backend --------------
		operatorStateBackend = operatorStateBackend(
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry);

		// -------------- Raw State Streams --------------
		rawKeyedStateInputs = rawKeyedStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs);

		rawOperatorStateInputs = rawOperatorStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs);

		// -------------- Internal Timer Service Manager --------------
		timeServiceManager = internalTimeServiceManager(keyedStatedBackend, keyContext, rawKeyedStateInputs);

		// -------------- Preparing return value --------------

		return new StreamOperatorStateContextImpl(
			prioritizedOperatorSubtaskStates.isRestored(),
			operatorStateBackend,
			keyedStatedBackend,
			timeServiceManager,
			rawOperatorStateInputs,
			rawKeyedStateInputs);
	} catch (Exception ex) {

		// cleanup if something went wrong before results got published.
		if (keyedStatedBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(keyedStatedBackend)) {
				IOUtils.closeQuietly(keyedStatedBackend);
			}
			// release resource (e.g native resource)
			keyedStatedBackend.dispose();
		}

		if (operatorStateBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(operatorStateBackend)) {
				IOUtils.closeQuietly(operatorStateBackend);
			}
			operatorStateBackend.dispose();
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawKeyedStateInputs)) {
			IOUtils.closeQuietly(rawKeyedStateInputs);
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawOperatorStateInputs)) {
			IOUtils.closeQuietly(rawOperatorStateInputs);
		}

		throw new Exception("Exception while creating StreamOperatorStateContext.", ex);
	}
}

Example 13

Source File: RocksDBStateBackendTest.java From flink with Apache License 2.0

4 votes

@Test
public void testSharedIncrementalStateDeRegistration() throws Exception {
	if (enableIncrementalCheckpointing) {
		AbstractKeyedStateBackend<Integer> backend = createKeyedBackend(IntSerializer.INSTANCE);
		try {
			ValueStateDescriptor<String> kvId =
				new ValueStateDescriptor<>("id", String.class, null);

			kvId.initializeSerializerUnlessSet(new ExecutionConfig());

			ValueState<String> state =
				backend.getPartitionedState(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, kvId);

			Queue<IncrementalRemoteKeyedStateHandle> previousStateHandles = new LinkedList<>();
			SharedStateRegistry sharedStateRegistry = spy(new SharedStateRegistry());
			for (int checkpointId = 0; checkpointId < 3; ++checkpointId) {

				reset(sharedStateRegistry);

				backend.setCurrentKey(checkpointId);
				state.update("Hello-" + checkpointId);

				RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot = backend.snapshot(
					checkpointId,
					checkpointId,
					createStreamFactory(),
					CheckpointOptions.forCheckpointWithDefaultLocation());

				snapshot.run();

				SnapshotResult<KeyedStateHandle> snapshotResult = snapshot.get();

				IncrementalRemoteKeyedStateHandle stateHandle =
					(IncrementalRemoteKeyedStateHandle) snapshotResult.getJobManagerOwnedSnapshot();

				Map<StateHandleID, StreamStateHandle> sharedState =
					new HashMap<>(stateHandle.getSharedState());

				stateHandle.registerSharedStates(sharedStateRegistry);

				for (Map.Entry<StateHandleID, StreamStateHandle> e : sharedState.entrySet()) {
					verify(sharedStateRegistry).registerReference(
						stateHandle.createSharedStateRegistryKeyFromFileName(e.getKey()),
						e.getValue());
				}

				previousStateHandles.add(stateHandle);
				backend.notifyCheckpointComplete(checkpointId);

				//-----------------------------------------------------------------

				if (previousStateHandles.size() > 1) {
					checkRemove(previousStateHandles.remove(), sharedStateRegistry);
				}
			}

			while (!previousStateHandles.isEmpty()) {

				reset(sharedStateRegistry);

				checkRemove(previousStateHandles.remove(), sharedStateRegistry);
			}
		} finally {
			IOUtils.closeQuietly(backend);
			backend.dispose();
		}
	}
}

Example 14

Source File: RocksDBAsyncSnapshotTest.java From flink with Apache License 2.0

4 votes

/**
 * Test that the snapshot files are cleaned up in case of a failure during the snapshot
 * procedure.
 */
@Test
public void testCleanupOfSnapshotsInFailureCase() throws Exception {
	long checkpointId = 1L;
	long timestamp = 42L;

	MockEnvironment env = MockEnvironment.builder().build();

	final IOException testException = new IOException("Test exception");
	CheckpointStateOutputStream outputStream = spy(new FailingStream(testException));

	RocksDBStateBackend backend = new RocksDBStateBackend((StateBackend) new MemoryStateBackend());

	backend.setDbStoragePath(temporaryFolder.newFolder().toURI().toString());

	AbstractKeyedStateBackend<Void> keyedStateBackend = backend.createKeyedStateBackend(
		env,
		new JobID(),
		"test operator",
		VoidSerializer.INSTANCE,
		1,
		new KeyGroupRange(0, 0),
		null,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		new CloseableRegistry());

	try {
		// register a state so that the state backend has to checkpoint something
		keyedStateBackend.getPartitionedState(
			"namespace",
			StringSerializer.INSTANCE,
			new ValueStateDescriptor<>("foobar", String.class));

		RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotFuture = keyedStateBackend.snapshot(
			checkpointId, timestamp,
			new TestCheckpointStreamFactory(() -> outputStream),
			CheckpointOptions.forCheckpointWithDefaultLocation());

		try {
			FutureUtils.runIfNotDoneAndGet(snapshotFuture);
			fail("Expected an exception to be thrown here.");
		} catch (ExecutionException e) {
			Assert.assertEquals(testException, e.getCause());
		}

		verify(outputStream).close();
	} finally {
		IOUtils.closeQuietly(keyedStateBackend);
		keyedStateBackend.dispose();
		IOUtils.closeQuietly(env);
	}
}

Example 15

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

4 votes

@Override
public StreamOperatorStateContext streamOperatorStateContext(
	@Nonnull OperatorID operatorID,
	@Nonnull String operatorClassName,
	@Nonnull ProcessingTimeService processingTimeService,
	@Nonnull KeyContext keyContext,
	@Nullable TypeSerializer<?> keySerializer,
	@Nonnull CloseableRegistry streamTaskCloseableRegistry,
	@Nonnull MetricGroup metricGroup) throws Exception {

	TaskInfo taskInfo = environment.getTaskInfo();
	OperatorSubtaskDescriptionText operatorSubtaskDescription =
		new OperatorSubtaskDescriptionText(
			operatorID,
			operatorClassName,
			taskInfo.getIndexOfThisSubtask(),
			taskInfo.getNumberOfParallelSubtasks());

	final String operatorIdentifierText = operatorSubtaskDescription.toString();

	final PrioritizedOperatorSubtaskState prioritizedOperatorSubtaskStates =
		taskStateManager.prioritizedOperatorState(operatorID);

	AbstractKeyedStateBackend<?> keyedStatedBackend = null;
	OperatorStateBackend operatorStateBackend = null;
	CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs = null;
	CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs = null;
	InternalTimeServiceManager<?> timeServiceManager;

	try {

		// -------------- Keyed State Backend --------------
		keyedStatedBackend = keyedStatedBackend(
			keySerializer,
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry,
			metricGroup);

		// -------------- Operator State Backend --------------
		operatorStateBackend = operatorStateBackend(
			operatorIdentifierText,
			prioritizedOperatorSubtaskStates,
			streamTaskCloseableRegistry);

		// -------------- Raw State Streams --------------
		rawKeyedStateInputs = rawKeyedStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawKeyedState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawKeyedStateInputs);

		rawOperatorStateInputs = rawOperatorStateInputs(
			prioritizedOperatorSubtaskStates.getPrioritizedRawOperatorState().iterator());
		streamTaskCloseableRegistry.registerCloseable(rawOperatorStateInputs);

		// -------------- Internal Timer Service Manager --------------
		timeServiceManager = internalTimeServiceManager(keyedStatedBackend, keyContext, processingTimeService, rawKeyedStateInputs);

		// -------------- Preparing return value --------------

		return new StreamOperatorStateContextImpl(
			prioritizedOperatorSubtaskStates.isRestored(),
			operatorStateBackend,
			keyedStatedBackend,
			timeServiceManager,
			rawOperatorStateInputs,
			rawKeyedStateInputs);
	} catch (Exception ex) {

		// cleanup if something went wrong before results got published.
		if (keyedStatedBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(keyedStatedBackend)) {
				IOUtils.closeQuietly(keyedStatedBackend);
			}
			// release resource (e.g native resource)
			keyedStatedBackend.dispose();
		}

		if (operatorStateBackend != null) {
			if (streamTaskCloseableRegistry.unregisterCloseable(operatorStateBackend)) {
				IOUtils.closeQuietly(operatorStateBackend);
			}
			operatorStateBackend.dispose();
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawKeyedStateInputs)) {
			IOUtils.closeQuietly(rawKeyedStateInputs);
		}

		if (streamTaskCloseableRegistry.unregisterCloseable(rawOperatorStateInputs)) {
			IOUtils.closeQuietly(rawOperatorStateInputs);
		}

		throw new Exception("Exception while creating StreamOperatorStateContext.", ex);
	}
}