org.apache.flink.runtime.state.CheckpointMetadataOutputStream Java Examples

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	storage.initializeBaseLocations();
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 

@Test
public void writeToAlreadyExistingCheckpointFails() throws Exception {
	final byte[] data = {8, 8, 4, 5, 2, 6, 3};
	final long checkpointId = 177;

	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());
	final CheckpointStorageLocation loc = storage.initializeLocationForCheckpoint(checkpointId);

	// write to the metadata file for the checkpoint

	try (CheckpointMetadataOutputStream out = loc.createMetadataOutputStream()) {
		out.write(data);
		out.closeAndFinalizeCheckpoint();
	}

	// create another writer to the metadata file for the checkpoint
	try {
		loc.createMetadataOutputStream();
		fail("this should fail with an exception");
	}
	catch (IOException ignored) {}
}
 

@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 

@Override
public void writeRecord(Savepoint savepoint) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(savepoint, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 

@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(metadata, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path checkpointDir = new Path(tmp.newFolder().toURI());

	final long checkpointId = 177;

	final CheckpointStorage storage1 = createCheckpointStorage(checkpointDir);
	storage1.initializeBaseLocations();
	final CheckpointStorage storage2 = createCheckpointStorage(checkpointDir);
	storage2.initializeBaseLocations();

	final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
	final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);

	final byte[] data1 = {77, 66, 55, 99, 88};
	final byte[] data2 = {1, 3, 2, 5, 4};

	final CompletedCheckpointStorageLocation completedLocation1;
	try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
		out.write(data1);
		completedLocation1 = out.closeAndFinalizeCheckpoint();
	}
	final String result1 = completedLocation1.getExternalPointer();

	final CompletedCheckpointStorageLocation completedLocation2;
	try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
		out.write(data2);
		completedLocation2 = out.closeAndFinalizeCheckpoint();
	}
	final String result2 = completedLocation2.getExternalPointer();

	// check that this went to a file, but in a nested directory structure

	// one directory per storage
	FileStatus[] files = fs.listStatus(checkpointDir);
	assertEquals(2, files.length);

	// in each per-storage directory, one for the checkpoint
	FileStatus[] job1Files = fs.listStatus(files[0].getPath());
	FileStatus[] job2Files = fs.listStatus(files[1].getPath());
	assertTrue(job1Files.length >= 1);
	assertTrue(job2Files.length >= 1);

	assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));
	assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));

	// check that both storages can resolve each others contents
	validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
	validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new MetadataOutputStream();
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new MetadataOutputStream();
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

public CompletedCheckpoint finalizeCheckpoint() throws IOException {

		synchronized (lock) {
			checkState(!isDiscarded(), "checkpoint is discarded");
			checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet");

			// make sure we fulfill the promise with an exception if something fails
			try {
				// write out the metadata
				final CheckpointMetadata savepoint = new CheckpointMetadata(checkpointId, operatorStates.values(), masterStates);
				final CompletedCheckpointStorageLocation finalizedLocation;

				try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
					Checkpoints.storeCheckpointMetadata(savepoint, out);
					finalizedLocation = out.closeAndFinalizeCheckpoint();
				}

				CompletedCheckpoint completed = new CompletedCheckpoint(
						jobId,
						checkpointId,
						checkpointTimestamp,
						System.currentTimeMillis(),
						operatorStates,
						masterStates,
						props,
						finalizedLocation);

				onCompletionPromise.complete(completed);

				// to prevent null-pointers from concurrent modification, copy reference onto stack
				PendingCheckpointStats statsCallback = this.statsCallback;
				if (statsCallback != null) {
					// Finalize the statsCallback and give the completed checkpoint a
					// callback for discards.
					CompletedCheckpointStats.DiscardCallback discardCallback =
							statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
					completed.setDiscardCallback(discardCallback);
				}

				// mark this pending checkpoint as disposed, but do NOT drop the state
				dispose(false);

				return completed;
			}
			catch (Throwable t) {
				onCompletionPromise.completeExceptionally(t);
				ExceptionUtils.rethrowIOException(t);
				return null; // silence the compiler
			}
		}
	}
 

/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path checkpointDir = new Path(tmp.newFolder().toURI());

	final long checkpointId = 177;

	final CheckpointStorage storage1 = createCheckpointStorage(checkpointDir);
	final CheckpointStorage storage2 = createCheckpointStorage(checkpointDir);

	final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
	final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);

	final byte[] data1 = {77, 66, 55, 99, 88};
	final byte[] data2 = {1, 3, 2, 5, 4};

	final CompletedCheckpointStorageLocation completedLocation1;
	try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
		out.write(data1);
		completedLocation1 = out.closeAndFinalizeCheckpoint();
	}
	final String result1 = completedLocation1.getExternalPointer();

	final CompletedCheckpointStorageLocation completedLocation2;
	try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
		out.write(data2);
		completedLocation2 = out.closeAndFinalizeCheckpoint();
	}
	final String result2 = completedLocation2.getExternalPointer();

	// check that this went to a file, but in a nested directory structure

	// one directory per storage
	FileStatus[] files = fs.listStatus(checkpointDir);
	assertEquals(2, files.length);

	// in each per-storage directory, one for the checkpoint
	FileStatus[] job1Files = fs.listStatus(files[0].getPath());
	FileStatus[] job2Files = fs.listStatus(files[1].getPath());
	assertTrue(job1Files.length >= 1);
	assertTrue(job2Files.length >= 1);

	assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));
	assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));

	// check that both storages can resolve each others contents
	validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
	validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}
 

/**
 * Validates that multiple checkpoints from different jobs with the same checkpoint ID do not
 * interfere with each other.
 */
@Test
public void testPersistMultipleMetadataOnlyCheckpoints() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path checkpointDir = new Path(tmp.newFolder().toURI());

	final long checkpointId = 177;

	final CheckpointStorage storage1 = createCheckpointStorage(checkpointDir);
	final CheckpointStorage storage2 = createCheckpointStorage(checkpointDir);

	final CheckpointStorageLocation loc1 = storage1.initializeLocationForCheckpoint(checkpointId);
	final CheckpointStorageLocation loc2 = storage2.initializeLocationForCheckpoint(checkpointId);

	final byte[] data1 = {77, 66, 55, 99, 88};
	final byte[] data2 = {1, 3, 2, 5, 4};

	final CompletedCheckpointStorageLocation completedLocation1;
	try (CheckpointMetadataOutputStream out = loc1.createMetadataOutputStream()) {
		out.write(data1);
		completedLocation1 = out.closeAndFinalizeCheckpoint();
	}
	final String result1 = completedLocation1.getExternalPointer();

	final CompletedCheckpointStorageLocation completedLocation2;
	try (CheckpointMetadataOutputStream out = loc2.createMetadataOutputStream()) {
		out.write(data2);
		completedLocation2 = out.closeAndFinalizeCheckpoint();
	}
	final String result2 = completedLocation2.getExternalPointer();

	// check that this went to a file, but in a nested directory structure

	// one directory per storage
	FileStatus[] files = fs.listStatus(checkpointDir);
	assertEquals(2, files.length);

	// in each per-storage directory, one for the checkpoint
	FileStatus[] job1Files = fs.listStatus(files[0].getPath());
	FileStatus[] job2Files = fs.listStatus(files[1].getPath());
	assertTrue(job1Files.length >= 1);
	assertTrue(job2Files.length >= 1);

	assertTrue(fs.exists(new Path(result1, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));
	assertTrue(fs.exists(new Path(result2, AbstractFsCheckpointStorage.METADATA_FILE_NAME)));

	// check that both storages can resolve each others contents
	validateContents(storage1.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage1.resolveCheckpoint(result2).getMetadataHandle(), data2);
	validateContents(storage2.resolveCheckpoint(result1).getMetadataHandle(), data1);
	validateContents(storage2.resolveCheckpoint(result2).getMetadataHandle(), data2);
}
 

public CompletedCheckpoint finalizeCheckpoint() throws IOException {

		synchronized (lock) {
			checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet.");

			// make sure we fulfill the promise with an exception if something fails
			try {
				// write out the metadata
				final Savepoint savepoint = new SavepointV2(checkpointId, operatorStates.values(), masterState);
				final CompletedCheckpointStorageLocation finalizedLocation;

				try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
					Checkpoints.storeCheckpointMetadata(savepoint, out);
					finalizedLocation = out.closeAndFinalizeCheckpoint();
				}

				CompletedCheckpoint completed = new CompletedCheckpoint(
						jobId,
						checkpointId,
						checkpointTimestamp,
						System.currentTimeMillis(),
						operatorStates,
						masterState,
						props,
						finalizedLocation);

				onCompletionPromise.complete(completed);

				// to prevent null-pointers from concurrent modification, copy reference onto stack
				PendingCheckpointStats statsCallback = this.statsCallback;
				if (statsCallback != null) {
					// Finalize the statsCallback and give the completed checkpoint a
					// callback for discards.
					CompletedCheckpointStats.DiscardCallback discardCallback =
							statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
					completed.setDiscardCallback(discardCallback);
				}

				// mark this pending checkpoint as disposed, but do NOT drop the state
				dispose(false);

				return completed;
			}
			catch (Throwable t) {
				onCompletionPromise.completeExceptionally(t);
				ExceptionUtils.rethrowIOException(t);
				return null; // silence the compiler
			}
		}
	}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new MetadataOutputStream();
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

@Override
public CheckpointMetadataOutputStream createMetadataOutputStream() throws IOException {
	return new FsCheckpointMetadataOutputStream(fileSystem, metadataFilePath, checkpointDirectory);
}
 

public CompletedCheckpoint finalizeCheckpoint() throws IOException {

		synchronized (lock) {
			checkState(isFullyAcknowledged(), "Pending checkpoint has not been fully acknowledged yet.");

			// make sure we fulfill the promise with an exception if something fails
			try {
				// write out the metadata
				final Savepoint savepoint = new SavepointV2(checkpointId, operatorStates.values(), masterState);
				final CompletedCheckpointStorageLocation finalizedLocation;

				try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
					Checkpoints.storeCheckpointMetadata(savepoint, out);
					finalizedLocation = out.closeAndFinalizeCheckpoint();
				}

				CompletedCheckpoint completed = new CompletedCheckpoint(
						jobId,
						checkpointId,
						checkpointTimestamp,
						System.currentTimeMillis(),
						operatorStates,
						masterState,
						props,
						finalizedLocation);

				onCompletionPromise.complete(completed);

				// to prevent null-pointers from concurrent modification, copy reference onto stack
				PendingCheckpointStats statsCallback = this.statsCallback;
				if (statsCallback != null) {
					// Finalize the statsCallback and give the completed checkpoint a
					// callback for discards.
					CompletedCheckpointStats.DiscardCallback discardCallback =
							statsCallback.reportCompletedCheckpoint(finalizedLocation.getExternalPointer());
					completed.setDiscardCallback(discardCallback);
				}

				// mark this pending checkpoint as disposed, but do NOT drop the state
				dispose(false);

				return completed;
			}
			catch (Throwable t) {
				onCompletionPromise.completeExceptionally(t);
				ExceptionUtils.rethrowIOException(t);
				return null; // silence the compiler
			}
		}
	}