Java Code Examples for org.apache.flink.runtime.state.CheckpointStorageLocationReference

The following examples show how to use org.apache.flink.runtime.state.CheckpointStorageLocationReference. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: FsCheckpointStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0);

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FsCheckpointStorageLocation.java    License: Apache License 2.0 6 votes vote down vote up
public FsCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		Path sharedStateDir,
		Path taskOwnedStateDir,
		CheckpointStorageLocationReference reference,
		int fileStateSizeThreshold) {

	super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold);

	checkArgument(fileStateSizeThreshold >= 0);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.sharedStateDirectory = checkNotNull(sharedStateDir);
	this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir);
	this.reference = checkNotNull(reference);

	// the metadata file should not have entropy in its path
	Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir);

	this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	this.fileStateSizeThreshold = fileStateSizeThreshold;
}
 
Example 3
Source Project: Flink-CEPplus   Source File: PendingCheckpointTest.java    License: Apache License 2.0 6 votes vote down vote up
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException {

		final Path checkpointDir = new Path(tmpFolder.newFolder().toURI());
		final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation(
				LocalFileSystem.getSharedInstance(),
				checkpointDir, checkpointDir, checkpointDir,
				CheckpointStorageLocationReference.getDefault(),
				1024);

		final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS);

		return new PendingCheckpoint(
			new JobID(),
			0,
			1,
			ackTasks,
			props,
			location,
			executor);
	}
 
Example 4
Source Project: Flink-CEPplus   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 5
Source Project: Flink-CEPplus   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example 6
@Test
public void testEncodeAndDecode() throws Exception {
	final Path path = randomPath(new Random());

	try {
		CheckpointStorageLocationReference ref = encodePathAsReference(path);
		Path decoded = decodePathFromReference(ref);

		assertEquals(path, decoded);
	}
	catch (Exception | Error e) {
		// if something goes wrong, help by printing the problematic path
		log.error("ERROR FOR PATH " + path);
		throw e;
	}
}
 
Example 7
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0);

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold,
			writeBufferSize);
}
 
Example 8
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
private boolean initCheckpoint(boolean unalignedCheckpointEnabled, CheckpointType checkpointType) throws IOException {
	class MockWriter extends ChannelStateWriterImpl.NoOpChannelStateWriter {
		private boolean started;
		@Override
		public void start(long checkpointId, CheckpointOptions checkpointOptions) {
			started = true;
		}
	}

	MockWriter writer = new MockWriter();
	SubtaskCheckpointCoordinator coordinator = coordinator(unalignedCheckpointEnabled, writer);
	CheckpointStorageLocationReference locationReference = CheckpointStorageLocationReference.getDefault();
	CheckpointOptions options = new CheckpointOptions(checkpointType, locationReference, true, unalignedCheckpointEnabled);
	coordinator.initCheckpoint(1L, options);
	return writer.started;
}
 
Example 9
Source Project: flink   Source File: PendingCheckpointTest.java    License: Apache License 2.0 6 votes vote down vote up
private PendingCheckpoint createPendingCheckpoint(CheckpointProperties props, Executor executor) throws IOException {

		final Path checkpointDir = new Path(tmpFolder.newFolder().toURI());
		final FsCheckpointStorageLocation location = new FsCheckpointStorageLocation(
				LocalFileSystem.getSharedInstance(),
				checkpointDir, checkpointDir, checkpointDir,
				CheckpointStorageLocationReference.getDefault(),
				1024,
				4096);

		final Map<ExecutionAttemptID, ExecutionVertex> ackTasks = new HashMap<>(ACK_TASKS);

		return new PendingCheckpoint(
			new JobID(),
			0,
			1,
			ackTasks,
			props,
			location,
			executor);
	}
 
Example 10
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that no mkdirs is called by the checkpoint storage location when resolved.
 */
@Test
public void testStorageLocationDoesNotMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir =  new File(storage.getCheckpointsDirectory().getPath());
	assertTrue(baseDir.exists());

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 11
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD,
		WRITE_BUFFER_SIZE);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example 12
Source Project: flink   Source File: FsStorageLocationReferenceTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEncodeAndDecode() throws Exception {
	final Path path = randomPath(new Random());

	try {
		CheckpointStorageLocationReference ref = encodePathAsReference(path);
		Path decoded = decodePathFromReference(ref);

		assertEquals(path, decoded);
	}
	catch (Exception | Error e) {
		// if something goes wrong, help by printing the problematic path
		log.error("ERROR FOR PATH " + path);
		throw e;
	}
}
 
Example 13
@Test
public void testOutOfOrderBarrier() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	long checkpointId = 10;
	long outOfOrderSavepointId = 5;
	long initialAlignedCheckpointId = alignedHandler.getLatestCheckpointId();

	barrierHandler.processBarrier(new CheckpointBarrier(checkpointId, 0, new CheckpointOptions(CHECKPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 0));
	barrierHandler.processBarrier(new CheckpointBarrier(outOfOrderSavepointId, 0, new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, 1));

	assertEquals(checkpointId, barrierHandler.getLatestCheckpointId());
	assertInflightDataEquals(unalignedHandler, barrierHandler, checkpointId, inputGate.getNumberOfInputChannels());
	assertEquals(initialAlignedCheckpointId, alignedHandler.getLatestCheckpointId());
}
 
Example 14
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResolveCheckpointStorageLocation() throws Exception {
	final FileSystem checkpointFileSystem = mock(FileSystem.class);
	final FsCheckpointStorage storage = new FsCheckpointStorage(
		new TestingPath("hdfs:///checkpoint/", checkpointFileSystem),
		null,
		new JobID(),
		FILE_SIZE_THRESHOLD,
		WRITE_BUFFER_SIZE);

	final FsCheckpointStorageLocation checkpointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(1L, CheckpointStorageLocationReference.getDefault());
	assertEquals(checkpointFileSystem, checkpointStreamFactory.getFileSystem());

	final CheckpointStorageLocationReference savepointLocationReference =
		AbstractFsCheckpointStorage.encodePathAsReference(new Path("file:///savepoint/"));

	final FsCheckpointStorageLocation savepointStreamFactory =
		(FsCheckpointStorageLocation) storage.resolveCheckpointStorageLocation(2L, savepointLocationReference);
	final FileSystem fileSystem = savepointStreamFactory.getFileSystem();
	assertTrue(fileSystem instanceof LocalFileSystem);
}
 
Example 15
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSkipChannelStateForSavepoints() throws Exception {
	SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder()
		.setUnalignedCheckpointEnabled(true)
		.setPrepareInputSnapshot((u1, u2) -> {
			fail("should not prepare input snapshot for savepoint");
			return null;
		}).build();

	coordinator.checkpointState(
		new CheckpointMetaData(0, 0),
		new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()),
		new CheckpointMetrics(),
		new OperatorChain<>(new NoOpStreamTask<>(new DummyEnvironment()), new NonRecordWriter<>()),
		() -> false);
}
 
Example 16
@Test
public void testPreviousHandlerReset() throws Exception {
	SingleInputGate inputGate = new SingleInputGateBuilder().setNumberOfChannels(2).build();
	inputGate.setInputChannels(new TestInputChannel(inputGate, 0), new TestInputChannel(inputGate, 1));
	TestInvokable target = new TestInvokable();
	CheckpointBarrierAligner alignedHandler = new CheckpointBarrierAligner("test", target, inputGate);
	CheckpointBarrierUnaligner unalignedHandler = new CheckpointBarrierUnaligner(TestSubtaskCheckpointCoordinator.INSTANCE, "test", target, inputGate);
	AlternatingCheckpointBarrierHandler barrierHandler = new AlternatingCheckpointBarrierHandler(alignedHandler, unalignedHandler, target);

	for (int i = 0; i < 4; i++) {
		int channel = i % 2;
		CheckpointType type = channel == 0 ? CHECKPOINT : SAVEPOINT;
		barrierHandler.processBarrier(new CheckpointBarrier(i, 0, new CheckpointOptions(type, CheckpointStorageLocationReference.getDefault())), new InputChannelInfo(0, channel));
		assertEquals(type.isSavepoint(), alignedHandler.isCheckpointPending());
		assertNotEquals(alignedHandler.isCheckpointPending(), unalignedHandler.isCheckpointPending());

		if (!type.isSavepoint()) {
			assertFalse(barrierHandler.getAllBarriersReceivedFuture(i).isDone());
			assertInflightDataEquals(unalignedHandler, barrierHandler, i, inputGate.getNumberOfInputChannels());
		}
	}
}
 
Example 17
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This test checks that the expected mkdirs action for checkpoint storage,
 * only be called when initializeBaseLocations and not called when resolveCheckpointStorageLocation.
 */
@Test
public void testStorageLocationMkdirs() throws Exception {
	FsCheckpointStorage storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	File baseDir = new File(storage.getCheckpointsDirectory().getPath());
	assertFalse(baseDir.exists());

	// mkdirs would only be called when initializeBaseLocations
	storage.initializeBaseLocations();
	assertTrue(baseDir.exists());

	// mkdir would not be called when resolveCheckpointStorageLocation
	storage = new FsCheckpointStorage(
			randomTempPath(), null, new JobID(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE);

	FsCheckpointStorageLocation location = (FsCheckpointStorageLocation)
			storage.resolveCheckpointStorageLocation(177, CheckpointStorageLocationReference.getDefault());

	Path checkpointPath = location.getCheckpointDirectory();
	File checkpointDir = new File(checkpointPath.getPath());
	assertFalse(checkpointDir.exists());
}
 
Example 18
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException {
	checkArgument(checkpointId >= 0, "Illegal negative checkpoint id: %d.", checkpointId);
	checkArgument(baseLocationsInitialized, "The base checkpoint location has not been initialized.");

	// prepare all the paths needed for the checkpoints
	final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

	// create the checkpoint exclusive directory
	fileSystem.mkdirs(checkpointDir);

	return new FsCheckpointStorageLocation(
			fileSystem,
			checkpointDir,
			sharedStateDirectory,
			taskOwnedStateDirectory,
			CheckpointStorageLocationReference.getDefault(),
			fileSizeThreshold,
			writeBufferSize);
}
 
Example 19
Source Project: flink   Source File: SubtaskCheckpointCoordinatorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSavepointNotResultingInPriorityEvents() throws Exception {
	MockEnvironment mockEnvironment = MockEnvironment.builder().build();

	SubtaskCheckpointCoordinator coordinator = new MockSubtaskCheckpointCoordinatorBuilder()
			.setUnalignedCheckpointEnabled(true)
			.setEnvironment(mockEnvironment)
			.build();

	AtomicReference<Boolean> broadcastedPriorityEvent = new AtomicReference<>(null);
	final OperatorChain<?, ?> operatorChain = new OperatorChain(
			new MockStreamTaskBuilder(mockEnvironment).build(),
			new NonRecordWriter<>()) {
		@Override
		public void broadcastEvent(AbstractEvent event, boolean isPriorityEvent) throws IOException {
			super.broadcastEvent(event, isPriorityEvent);
			broadcastedPriorityEvent.set(isPriorityEvent);
		}
	};

	coordinator.checkpointState(
			new CheckpointMetaData(0, 0),
			new CheckpointOptions(SAVEPOINT, CheckpointStorageLocationReference.getDefault()),
			new CheckpointMetrics(),
			operatorChain,
			() -> false);

	assertEquals(false, broadcastedPriorityEvent.get());
}
 
Example 20
Source Project: Flink-CEPplus   Source File: AbstractFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Encodes the given path as a reference in bytes. The path is encoded as a UTF-8 string
 * and prepended as a magic number.
 *
 * @param path The path to encode.
 * @return The location reference.
 */
public static CheckpointStorageLocationReference encodePathAsReference(Path path) {
	byte[] refBytes = path.toString().getBytes(StandardCharsets.UTF_8);
	byte[] bytes = new byte[REFERENCE_MAGIC_NUMBER.length + refBytes.length];

	System.arraycopy(REFERENCE_MAGIC_NUMBER, 0, bytes, 0, REFERENCE_MAGIC_NUMBER.length);
	System.arraycopy(refBytes, 0, bytes, REFERENCE_MAGIC_NUMBER.length, refBytes.length);

	return new CheckpointStorageLocationReference(bytes);
}
 
Example 21
Source Project: Flink-CEPplus   Source File: AbstractFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Decodes the given reference into a path. This method validates that the reference bytes start with
 * the correct magic number (as written by {@link #encodePathAsReference(Path)}) and converts
 * the remaining bytes back to a proper path.
 *
 * @param reference The bytes representing the reference.
 * @return The path decoded from the reference.
 *
 * @throws IllegalArgumentException Thrown, if the bytes do not represent a proper reference.
 */
public static Path decodePathFromReference(CheckpointStorageLocationReference reference) {
	if (reference.isDefaultReference()) {
		throw new IllegalArgumentException("Cannot decode default reference");
	}

	final byte[] bytes = reference.getReferenceBytes();
	final int headerLen = REFERENCE_MAGIC_NUMBER.length;

	if (bytes.length > headerLen) {
		// compare magic number
		for (int i = 0; i < headerLen; i++) {
			if (bytes[i] != REFERENCE_MAGIC_NUMBER[i]) {
				throw new IllegalArgumentException("Reference starts with the wrong magic number");
			}
		}

		// covert to string and path
		try {
			return new Path(new String(bytes, headerLen, bytes.length - headerLen, StandardCharsets.UTF_8));
		}
		catch (Exception e) {
			throw new IllegalArgumentException("Reference cannot be decoded to a path", e);
		}
	}
	else {
		throw new IllegalArgumentException("Reference too short.");
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStreamFactory resolveCheckpointStorageLocation(
		long checkpointId,
		CheckpointStorageLocationReference reference) throws IOException {

	if (reference.isDefaultReference()) {
		// default reference, construct the default location for that particular checkpoint
		final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

		return new FsCheckpointStorageLocation(
				fileSystem,
				checkpointDir,
				sharedStateDirectory,
				taskOwnedStateDirectory,
				reference,
				fileSizeThreshold);
	}
	else {
		// location encoded in the reference
		final Path path = decodePathFromReference(reference);

		return new FsCheckpointStorageLocation(
				path.getFileSystem(),
				path,
				path,
				path,
				reference,
				fileSizeThreshold);
	}
}
 
Example 23
Source Project: Flink-CEPplus   Source File: EventSerializerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckpointBarrierSerialization() throws Exception {
	long id = Integer.MAX_VALUE + 123123L;
	long timestamp = Integer.MAX_VALUE + 1228L;

	CheckpointOptions checkpoint = CheckpointOptions.forCheckpointWithDefaultLocation();
	testCheckpointBarrierSerialization(id, timestamp, checkpoint);

	final byte[] reference = new byte[] { 15, 52, 52, 11, 0, 0, 0, 0, -1, -23, -19, 35 };

	CheckpointOptions savepoint = new CheckpointOptions(
			CheckpointType.SAVEPOINT, new CheckpointStorageLocationReference(reference));
	testCheckpointBarrierSerialization(id, timestamp, savepoint);
}
 
Example 24
/**
 * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory)}.
 */
public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp) throws Exception {

	OperatorSnapshotFutures operatorStateResult = operator.snapshotState(
		checkpointId,
		timestamp,
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		checkpointStorage.resolveCheckpointStorageLocation(checkpointId, CheckpointStorageLocationReference.getDefault()));

	return new OperatorSnapshotFinalizer(operatorStateResult);
}
 
Example 25
Source Project: flink   Source File: SavepointOutputFormat.java    License: Apache License 2.0 5 votes vote down vote up
private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 
Example 26
Source Project: flink   Source File: AbstractStreamOperatorTestHarness.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calls {@link StreamOperator#snapshotState(long, long, CheckpointOptions, org.apache.flink.runtime.state.CheckpointStreamFactory)}.
 */
public OperatorSnapshotFinalizer snapshotWithLocalState(long checkpointId, long timestamp) throws Exception {

	OperatorSnapshotFutures operatorStateResult = operator.snapshotState(
		checkpointId,
		timestamp,
		CheckpointOptions.forCheckpointWithDefaultLocation(),
		checkpointStorage.resolveCheckpointStorageLocation(checkpointId, CheckpointStorageLocationReference.getDefault()));

	return new OperatorSnapshotFinalizer(operatorStateResult);
}
 
Example 27
Source Project: flink   Source File: AbstractFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Encodes the given path as a reference in bytes. The path is encoded as a UTF-8 string
 * and prepended as a magic number.
 *
 * @param path The path to encode.
 * @return The location reference.
 */
public static CheckpointStorageLocationReference encodePathAsReference(Path path) {
	byte[] refBytes = path.toString().getBytes(StandardCharsets.UTF_8);
	byte[] bytes = new byte[REFERENCE_MAGIC_NUMBER.length + refBytes.length];

	System.arraycopy(REFERENCE_MAGIC_NUMBER, 0, bytes, 0, REFERENCE_MAGIC_NUMBER.length);
	System.arraycopy(refBytes, 0, bytes, REFERENCE_MAGIC_NUMBER.length, refBytes.length);

	return new CheckpointStorageLocationReference(bytes);
}
 
Example 28
Source Project: flink   Source File: AbstractFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Decodes the given reference into a path. This method validates that the reference bytes start with
 * the correct magic number (as written by {@link #encodePathAsReference(Path)}) and converts
 * the remaining bytes back to a proper path.
 *
 * @param reference The bytes representing the reference.
 * @return The path decoded from the reference.
 *
 * @throws IllegalArgumentException Thrown, if the bytes do not represent a proper reference.
 */
public static Path decodePathFromReference(CheckpointStorageLocationReference reference) {
	if (reference.isDefaultReference()) {
		throw new IllegalArgumentException("Cannot decode default reference");
	}

	final byte[] bytes = reference.getReferenceBytes();
	final int headerLen = REFERENCE_MAGIC_NUMBER.length;

	if (bytes.length > headerLen) {
		// compare magic number
		for (int i = 0; i < headerLen; i++) {
			if (bytes[i] != REFERENCE_MAGIC_NUMBER[i]) {
				throw new IllegalArgumentException("Reference starts with the wrong magic number");
			}
		}

		// covert to string and path
		try {
			return new Path(new String(bytes, headerLen, bytes.length - headerLen, StandardCharsets.UTF_8));
		}
		catch (Exception e) {
			throw new IllegalArgumentException("Reference cannot be decoded to a path", e);
		}
	}
	else {
		throw new IllegalArgumentException("Reference too short.");
	}
}
 
Example 29
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStreamFactory resolveCheckpointStorageLocation(
		long checkpointId,
		CheckpointStorageLocationReference reference) throws IOException {

	if (reference.isDefaultReference()) {
		// default reference, construct the default location for that particular checkpoint
		final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId);

		return new FsCheckpointStorageLocation(
				fileSystem,
				checkpointDir,
				sharedStateDirectory,
				taskOwnedStateDirectory,
				reference,
				fileSizeThreshold,
				writeBufferSize);
	}
	else {
		// location encoded in the reference
		final Path path = decodePathFromReference(reference);

		return new FsCheckpointStorageLocation(
				path.getFileSystem(),
				path,
				path,
				path,
				reference,
				fileSizeThreshold,
				writeBufferSize);
	}
}
 
Example 30
Source Project: flink   Source File: FsCheckpointStorageLocation.java    License: Apache License 2.0 5 votes vote down vote up
public FsCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		Path sharedStateDir,
		Path taskOwnedStateDir,
		CheckpointStorageLocationReference reference,
		int fileStateSizeThreshold,
		int writeBufferSize) {

	super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold, writeBufferSize);

	checkArgument(fileStateSizeThreshold >= 0);
	checkArgument(writeBufferSize >= 0);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.sharedStateDirectory = checkNotNull(sharedStateDir);
	this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir);
	this.reference = checkNotNull(reference);

	// the metadata file should not have entropy in its path
	Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir);

	this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	this.fileStateSizeThreshold = fileStateSizeThreshold;
	this.writeBufferSize = writeBufferSize;
}