org.apache.flink.runtime.state.CompletedCheckpointStorageLocation Java Examples

The following examples show how to use org.apache.flink.runtime.state.CompletedCheckpointStorageLocation. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 6 votes vote down vote up
/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 
Example #2
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that savepoint loading fails when there is non-restored state, but it is not allowed.
 */
@Test
public void testNonRestoredStateWhenDisallowed() throws Exception {
	final OperatorID operatorId = new OperatorID();
	final int parallelism = 9;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(242L, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = Collections.emptyMap();

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("allowNonRestoredState"));
	}
}
 
Example #3
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that savepoint loading fails when there is non-restored coordinator state only,
 * and non-restored state is not allowed.
 */
@Test
public void testUnmatchedCoordinatorOnlyStateFails() throws Exception {
	final OperatorID operatorID = new OperatorID();
	final int maxParallelism = 1234;

	final OperatorState state = new OperatorState(operatorID, maxParallelism / 2, maxParallelism);
	state.setCoordinatorState(new ByteStreamStateHandle("coordinatorState", new byte[0]));

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorState(42L, state);
	final Map<JobVertexID, ExecutionJobVertex> tasks = Collections.emptyMap();

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("allowNonRestoredState"));
	}
}
 
Example #4
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static CompletedCheckpointStorageLocation createSavepointWithOperatorSubtaskState(
		final long checkpointId,
		final OperatorID operatorId,
		final int parallelism) throws IOException {

	final Random rnd = new Random();

	final OperatorSubtaskState subtaskState = new OperatorSubtaskState(
		new OperatorStreamStateHandle(Collections.emptyMap(), new ByteStreamStateHandle("testHandler", new byte[0])),
		null,
		null,
		null,
		singleton(createNewInputChannelStateHandle(10, rnd)),
		singleton(createNewResultSubpartitionStateHandle(10, rnd)));

	final OperatorState state = new OperatorState(operatorId, parallelism, parallelism);
	state.putState(0, subtaskState);

	return createSavepointWithOperatorState(checkpointId, state);
}
 
Example #5
Source File: DispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new CheckpointMetadata(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 
Example #6
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that savepoint loading fails when there is a max-parallelism mismatch.
 */
@Test
public void testMaxParallelismMismatch() throws Exception {
	final OperatorID operatorId = new OperatorID();
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(242L, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism + 1);

	try {
		Checkpoints.loadAndValidateCheckpoint(new JobID(), tasks, testSavepoint, cl, false);
		fail("Did not throw expected Exception");
	} catch (IllegalStateException expected) {
		assertTrue(expected.getMessage().contains("Max parallelism mismatch"));
	}
}
 
Example #7
Source File: CheckpointMetadataLoadingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests correct savepoint loading.
 */
@Test
public void testAllStateRestored() throws Exception {
	final JobID jobId = new JobID();
	final OperatorID operatorId = new OperatorID();
	final long checkpointId = Integer.MAX_VALUE + 123123L;
	final int parallelism = 128128;

	final CompletedCheckpointStorageLocation testSavepoint = createSavepointWithOperatorSubtaskState(checkpointId, operatorId, parallelism);
	final Map<JobVertexID, ExecutionJobVertex> tasks = createTasks(operatorId, parallelism, parallelism);

	final CompletedCheckpoint loaded = Checkpoints.loadAndValidateCheckpoint(jobId, tasks, testSavepoint, cl, false);

	assertEquals(jobId, loaded.getJobId());
	assertEquals(checkpointId, loaded.getCheckpointID());
}
 
Example #8
Source File: MemoryCheckpointStorageTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #9
Source File: MemoryCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #10
Source File: DispatcherTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorage checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 
Example #11
Source File: DispatcherTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private URI createTestingSavepoint() throws IOException, URISyntaxException {
	final StateBackend stateBackend = Checkpoints.loadStateBackend(configuration, Thread.currentThread().getContextClassLoader(), log);
	final CheckpointStorageCoordinatorView checkpointStorage = stateBackend.createCheckpointStorage(jobGraph.getJobID());
	final File savepointFile = temporaryFolder.newFolder();
	final long checkpointId = 1L;

	final CheckpointStorageLocation checkpointStorageLocation = checkpointStorage.initializeLocationForSavepoint(checkpointId, savepointFile.getAbsolutePath());

	final CheckpointMetadataOutputStream metadataOutputStream = checkpointStorageLocation.createMetadataOutputStream();
	Checkpoints.storeCheckpointMetadata(new SavepointV2(checkpointId, Collections.emptyList(), Collections.emptyList()), metadataOutputStream);

	final CompletedCheckpointStorageLocation completedCheckpointStorageLocation = metadataOutputStream.closeAndFinalizeCheckpoint();

	return new URI(completedCheckpointStorageLocation.getExternalPointer());

}
 
Example #12
Source File: MemoryCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #13
Source File: BravoTestPipeline.java    From bravo with Apache License 2.0 6 votes vote down vote up
public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 
Example #14
Source File: AbstractFileCheckpointStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example #15
Source File: AbstractFileCheckpointStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example #16
Source File: AbstractFileCheckpointStorageTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example #17
Source File: AbstractFileCheckpointStorageTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example #18
Source File: NonPersistentMetadataCheckpointStorageLocation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}
 
Example #19
Source File: CheckpointCoordinator.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
}
 
Example #20
Source File: CompletedCheckpoint.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 
Example #21
Source File: SavepointOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void writeRecord(Savepoint savepoint) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(savepoint, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 
Example #22
Source File: SavepointOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public void writeRecord(CheckpointMetadata metadata) throws IOException {
	String path = LambdaUtil.withContextClassLoader(getRuntimeContext().getUserCodeClassLoader(), () -> {
			try (CheckpointMetadataOutputStream out = targetLocation.createMetadataOutputStream()) {
				Checkpoints.storeCheckpointMetadata(metadata, out);
				CompletedCheckpointStorageLocation finalizedLocation = out.closeAndFinalizeCheckpoint();
				return finalizedLocation.getExternalPointer();
			}
	});

	LOG.info("Savepoint written to " + path);
}
 
Example #23
Source File: CheckpointCoordinator.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedState(tasks, true, allowNonRestored);
}
 
Example #24
Source File: CheckpointCoordinator.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Restore the state with given savepoint.
 *
 * @param savepointPointer The pointer to the savepoint.
 * @param allowNonRestored True if allowing checkpoint state that cannot be
 *                         mapped to any job vertex in tasks.
 * @param tasks            Map of job vertices to restore. State for these
 *                         vertices is restored via
 *                         {@link Execution#setInitialState(JobManagerTaskRestore)}.
 * @param userClassLoader  The class loader to resolve serialized classes in
 *                         legacy savepoint versions.
 */
public boolean restoreSavepoint(
		String savepointPointer,
		boolean allowNonRestored,
		Map<JobVertexID, ExecutionJobVertex> tasks,
		ClassLoader userClassLoader) throws Exception {

	Preconditions.checkNotNull(savepointPointer, "The savepoint path cannot be null.");

	LOG.info("Starting job {} from savepoint {} ({})",
			job, savepointPointer, (allowNonRestored ? "allowing non restored state" : ""));

	final CompletedCheckpointStorageLocation checkpointLocation = checkpointStorage.resolveCheckpoint(savepointPointer);

	// Load the savepoint as a checkpoint into the system
	CompletedCheckpoint savepoint = Checkpoints.loadAndValidateCheckpoint(
			job, tasks, checkpointLocation, userClassLoader, allowNonRestored);

	completedCheckpointStore.addCheckpoint(savepoint);

	// Reset the checkpoint ID counter
	long nextCheckpointId = savepoint.getCheckpointID() + 1;
	checkpointIdCounter.setCount(nextCheckpointId);

	LOG.info("Reset the checkpoint ID of job {} to {}.", job, nextCheckpointId);

	return restoreLatestCheckpointedStateInternal(new HashSet<>(tasks.values()), true, true, allowNonRestored);
}
 
Example #25
Source File: CompletedCheckpoint.java    From flink with Apache License 2.0 5 votes vote down vote up
public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 
Example #26
Source File: CompletedCheckpoint.java    From flink with Apache License 2.0 5 votes vote down vote up
public CompletedCheckpoint(
		JobID job,
		long checkpointID,
		long timestamp,
		long completionTimestamp,
		Map<OperatorID, OperatorState> operatorStates,
		@Nullable Collection<MasterState> masterHookStates,
		CheckpointProperties props,
		CompletedCheckpointStorageLocation storageLocation) {

	checkArgument(checkpointID >= 0);
	checkArgument(timestamp >= 0);
	checkArgument(completionTimestamp >= 0);

	this.job = checkNotNull(job);
	this.checkpointID = checkpointID;
	this.timestamp = timestamp;
	this.duration = completionTimestamp - timestamp;

	// we create copies here, to make sure we have no shared mutable
	// data structure with the "outside world"
	this.operatorStates = new HashMap<>(checkNotNull(operatorStates));
	this.masterHookStates = masterHookStates == null || masterHookStates.isEmpty() ?
			Collections.emptyList() :
			new ArrayList<>(masterHookStates);

	this.props = checkNotNull(props);
	this.storageLocation = checkNotNull(storageLocation);
	this.metadataHandle = storageLocation.getMetadataHandle();
	this.externalPointer = storageLocation.getExternalPointer();
}
 
Example #27
Source File: AbstractFileCheckpointStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example #28
Source File: AbstractFileCheckpointStorageTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example #29
Source File: NonPersistentMetadataCheckpointStorageLocation.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}
 
Example #30
Source File: NonPersistentMetadataCheckpointStorageLocation.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public CompletedCheckpointStorageLocation closeAndFinalizeCheckpoint() throws IOException {
	synchronized (this) {
		if (!closed) {
			closed = true;

			byte[] bytes = os.toByteArray();
			ByteStreamStateHandle handle = new ByteStreamStateHandle(UUID.randomUUID().toString(), bytes);
			return new NonPersistentCompletedCheckpointStorageLocation(handle);
		} else {
			throw new IOException("Already closed");
		}
	}
}