Java Code Examples for org.apache.flink.configuration.CheckpointingOptions

The following examples show how to use org.apache.flink.configuration.CheckpointingOptions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 3
Source Project: Flink-CEPplus   Source File: SavepointMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	return config;
}
 
Example 4
Source Project: Flink-CEPplus   Source File: FsStateBackendFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testConfigureMemoryStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointPath = new Path(appCheckpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final MemoryStateBackend backend = new MemoryStateBackend(appCheckpointDir, null);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this parameter should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
}
 
Example 6
@Test
public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {

	final int negativeMaxNumberOfCheckpointsToRetain = -10;

	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		negativeMaxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 
Example 7
Source Project: flink   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testConfigureMemoryStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointPath = new Path(appCheckpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final MemoryStateBackend backend = new MemoryStateBackend(appCheckpointDir, null);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this parameter should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
}
 
Example 8
Source Project: flink   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 9
Source Project: flink   Source File: SavepointMigrationTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	return config;
}
 
Example 10
Source Project: flink   Source File: LocalRecoveryITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public final void executeTest() throws Exception {
	EventTimeWindowCheckpointingITCase.tempFolder.create();
	EventTimeWindowCheckpointingITCase windowChkITCase =
		new EventTimeWindowCheckpointingITCase() {

			@Override
			protected StateBackendEnum getStateBackend() {
				return backendEnum;
			}

			@Override
			protected Configuration createClusterConfig() throws IOException {
				Configuration config = super.createClusterConfig();

				config.setBoolean(
					CheckpointingOptions.LOCAL_RECOVERY,
					localRecoveryEnabled);

				return config;
			}
		};

	executeTest(windowChkITCase);
}
 
Example 11
Source Project: flink   Source File: FsStateBackendFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 
Example 12
Source Project: flink   Source File: SavepointHandlers.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CompletableFuture<String> triggerOperation(
		final HandlerRequest<StopWithSavepointRequestBody, SavepointTriggerMessageParameters> request,
		final RestfulGateway gateway) throws RestHandlerException {

	final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
	final String requestedTargetDirectory = request.getRequestBody().getTargetDirectory();

	if (requestedTargetDirectory == null && defaultSavepointDir == null) {
		throw new RestHandlerException(
				String.format("Config key [%s] is not set. Property [%s] must be provided.",
						CheckpointingOptions.SAVEPOINT_DIRECTORY.key(),
						StopWithSavepointRequestBody.FIELD_NAME_TARGET_DIRECTORY),
				HttpResponseStatus.BAD_REQUEST);
	}

	final boolean advanceToEndOfEventTime = request.getRequestBody().shouldDrain();
	final String targetDirectory = requestedTargetDirectory != null ? requestedTargetDirectory : defaultSavepointDir;
	return gateway.stopWithSavepoint(jobId, targetDirectory, advanceToEndOfEventTime, RpcUtils.INF_TIMEOUT);
}
 
Example 13
Source Project: flink   Source File: ExecutionGraphDeploymentTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {

	final int negativeMaxNumberOfCheckpointsToRetain = -10;

	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		negativeMaxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 
Example 14
Source Project: flink   Source File: ExecutionGraphDeploymentTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSettingDefaultMaxNumberOfCheckpointsToRetain() throws Exception {
	final Configuration jobManagerConfig = new Configuration();

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 
Example 15
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerSavepointForNonExistingJob() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobID jobID = new JobID();

	try {
		client.triggerSavepoint(jobID, null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, jobID.toString()).isPresent());
	} finally {
		cluster.after();
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
	if (data == null) {
		// We need this to be large, because we want to test with files
		Random rand = new Random(getRuntimeContext().getIndexOfThisSubtask());
		data = new byte[CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue() + 1];
		rand.nextBytes(data);
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
private Configuration getFileBasedCheckpointsConfig(final String savepointDir) {
	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	return config;
}
 
Example 18
Source Project: flink   Source File: ExecutionGraphDeploymentTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSettingMaxNumberOfCheckpointsToRetain() throws Exception {

	final int maxNumberOfCheckpointsToRetain = 10;
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		maxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertEquals(maxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 
Example 19
Source Project: Flink-CEPplus   Source File: SchedulingITCase.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that if local recovery is disabled we won't spread
 * out tasks when recovering.
 */
@Test
public void testDisablingLocalRecovery() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, false);

	executeSchedulingTest(configuration);
}
 
Example 20
Source Project: Flink-CEPplus   Source File: SchedulingITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void testLocalRecoveryInternal(String failoverStrategyValue) throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(EXECUTION_FAILOVER_STRATEGY.key(), failoverStrategyValue);

	executeSchedulingTest(configuration);
}
 
Example 21
Source Project: flink   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration.
 */
@Test
public void testConfigureMemoryStateBackend() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointPath = new Path(checkpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final int maxSize = 100;
	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	final MemoryStateBackend backend = new MemoryStateBackend(maxSize, async);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, !async);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
	assertEquals(maxSize, memBackend.getMaxStateSize());
	assertEquals(async, memBackend.isUsingAsynchronousSnapshots());
}
 
Example 22
Source Project: Flink-CEPplus   Source File: RocksDBStateBackendFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RocksDBStateBackend createFromConfig(Configuration config, ClassLoader classLoader)
		throws IllegalConfigurationException, IOException {

	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDirURI = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDirURI == null) {
		throw new IllegalConfigurationException(
			"Cannot create the RocksDB state backend: The configuration does not specify the " +
			"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	return new RocksDBStateBackend(checkpointDirURI).configure(config, classLoader);
}
 
Example 23
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testStateBackendClosingOnFailure() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = true;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FAILED, task.getExecutionState());
}
 
Example 24
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final String localDir1 = tmp.newFolder().getAbsolutePath();
	final String localDir2 = tmp.newFolder().getAbsolutePath();
	final String localDir3 = tmp.newFolder().getAbsolutePath();
	final String localDir4 = tmp.newFolder().getAbsolutePath();

	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final Path expectedCheckpointsPath = new Path(appCheckpointDir);
	final Path expectedSavepointsPath = new Path(savepointDir);

	final RocksDBStateBackend backend = new RocksDBStateBackend(appCheckpointDir, incremental);
	backend.setDbStoragePaths(localDir1, localDir2);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setBoolean(CheckpointingOptions.INCREMENTAL_CHECKPOINTS, !incremental);  // this should not be picked up
	config.setString(RocksDBOptions.LOCAL_DIRECTORIES, localDir3 + ":" + localDir4);  // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof RocksDBStateBackend);

	final RocksDBStateBackend loadedRocks = (RocksDBStateBackend) loadedBackend;

	assertEquals(incremental, loadedRocks.isIncrementalCheckpointsEnabled());
	checkPaths(loadedRocks.getDbStoragePaths(), localDir1, localDir2);

	AbstractFileStateBackend fsBackend = (AbstractFileStateBackend) loadedRocks.getCheckpointBackend();
	assertEquals(expectedCheckpointsPath, fsBackend.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fsBackend.getSavepointPath());
}
 
Example 25
@Test
public void testDefaultsInSync() throws Exception {
	final boolean defaultIncremental = CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	RocksDBStateBackend backend = new RocksDBStateBackend(tempFolder.newFolder().toURI());
	assertEquals(defaultIncremental, backend.isIncrementalCheckpointsEnabled());
}
 
Example 26
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 
Example 27
Source Project: Flink-CEPplus   Source File: DefaultSchedulerFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private static SlotSelectionStrategy selectSlotSelectionStrategy(@Nonnull Configuration configuration) {
	if (configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY)) {
		return PreviousAllocationSlotSelectionStrategy.INSTANCE;
	} else {
		return LocationPreferenceSlotSelectionStrategy.INSTANCE;
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: JobMaster.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<String> triggerSavepoint(
		@Nullable final String targetDirectory,
		final boolean cancelJob,
		final Time timeout) {

	final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
	if (checkpointCoordinator == null) {
		return FutureUtils.completedExceptionally(new IllegalStateException(
			String.format("Job %s is not a streaming job.", jobGraph.getJobID())));
	} else if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) {
		log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID());

		return FutureUtils.completedExceptionally(new IllegalStateException(
			"No savepoint directory configured. You can either specify a directory " +
				"while cancelling via -s :targetDirectory or configure a cluster-wide " +
				"default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'."));
	}

	if (cancelJob) {
		checkpointCoordinator.stopCheckpointScheduler();
	}
	return checkpointCoordinator
		.triggerSavepoint(System.currentTimeMillis(), targetDirectory)
		.thenApply(CompletedCheckpoint::getExternalPointer)
		.handleAsync((path, throwable) -> {
			if (throwable != null) {
				if (cancelJob) {
					startCheckpointScheduler(checkpointCoordinator);
				}
				throw new CompletionException(throwable);
			} else if (cancelJob) {
				log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID());
				cancel(timeout);
			}
			return path;
		}, getMainThreadExecutor());
}
 
Example 29
Source Project: Flink-CEPplus   Source File: FsStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Private constructor that creates a re-configured copy of the state backend.
 *
 * @param original The state backend to re-configure
 * @param configuration The configuration
 */
private FsStateBackend(FsStateBackend original, Configuration configuration, ClassLoader classLoader) {
	super(original.getCheckpointPath(), original.getSavepointPath(), configuration);

	// if asynchronous snapshots were configured, use that setting,
	// else check the configuration
	this.asynchronousSnapshots = original.asynchronousSnapshots.resolveUndefined(
			configuration.getBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS));

	final int sizeThreshold = original.fileStateThreshold >= 0 ?
			original.fileStateThreshold :
			configuration.getInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD);

	if (sizeThreshold >= 0 && sizeThreshold <= MAX_FILE_STATE_THRESHOLD) {
		this.fileStateThreshold = sizeThreshold;
	}
	else {
		this.fileStateThreshold = CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue();

		// because this is the only place we (unlikely) ever log, we lazily
		// create the logger here
		LoggerFactory.getLogger(AbstractFileStateBackend.class).warn(
				"Ignoring invalid file size threshold value ({}): {} - using default value {} instead.",
				CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.key(), sizeThreshold,
				CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue());
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: MemoryStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Private constructor that creates a re-configured copy of the state backend.
 *
 * @param original The state backend to re-configure
 * @param configuration The configuration
 * @param classLoader The class loader
 */
private MemoryStateBackend(MemoryStateBackend original, Configuration configuration, ClassLoader classLoader) {
	super(original.getCheckpointPath(), original.getSavepointPath(), configuration);

	this.maxStateSize = original.maxStateSize;

	// if asynchronous snapshots were configured, use that setting,
	// else check the configuration
	this.asynchronousSnapshots = original.asynchronousSnapshots.resolveUndefined(
			configuration.getBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS));
}