org.apache.flink.configuration.CheckpointingOptions Java Examples

@Test
public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {

	final int negativeMaxNumberOfCheckpointsToRetain = -10;

	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		negativeMaxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 

@Override
protected CompletableFuture<String> triggerOperation(
		final HandlerRequest<StopWithSavepointRequestBody, SavepointTriggerMessageParameters> request,
		final RestfulGateway gateway) throws RestHandlerException {

	final JobID jobId = request.getPathParameter(JobIDPathParameter.class);
	final String requestedTargetDirectory = request.getRequestBody().getTargetDirectory();

	if (requestedTargetDirectory == null && defaultSavepointDir == null) {
		throw new RestHandlerException(
				String.format("Config key [%s] is not set. Property [%s] must be provided.",
						CheckpointingOptions.SAVEPOINT_DIRECTORY.key(),
						StopWithSavepointRequestBody.FIELD_NAME_TARGET_DIRECTORY),
				HttpResponseStatus.BAD_REQUEST);
	}

	final boolean advanceToEndOfEventTime = request.getRequestBody().shouldDrain();
	final String targetDirectory = requestedTargetDirectory != null ? requestedTargetDirectory : defaultSavepointDir;
	return gateway.stopWithSavepoint(jobId, targetDirectory, advanceToEndOfEventTime, RpcUtils.INF_TIMEOUT);
}
 

@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 

@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 

private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	return config;
}
 

@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 

/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testConfigureMemoryStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointPath = new Path(appCheckpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final MemoryStateBackend backend = new MemoryStateBackend(appCheckpointDir, null);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this parameter should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
}
 

private Configuration getConfiguration() throws Exception {
	// Flink configuration
	final Configuration config = new Configuration();

	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, 1);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, DEFAULT_PARALLELISM);

	UUID id = UUID.randomUUID();
	final File checkpointDir = TEMP_FOLDER.newFolder("checkpoints_" + id).getAbsoluteFile();
	final File savepointDir = TEMP_FOLDER.newFolder("savepoints_" + id).getAbsoluteFile();

	if (!checkpointDir.exists() || !savepointDir.exists()) {
		throw new Exception("Test setup failed: failed to create (temporary) directories.");
	}

	LOG.info("Created temporary checkpoint directory: " + checkpointDir + ".");
	LOG.info("Created savepoint directory: " + savepointDir + ".");

	config.setString(CheckpointingOptions.STATE_BACKEND, "memory");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	return config;
}
 

@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 

@Test
public final void executeTest() throws Exception {
	EventTimeWindowCheckpointingITCase.tempFolder.create();
	EventTimeWindowCheckpointingITCase windowChkITCase =
		new EventTimeWindowCheckpointingITCase() {

			@Override
			protected StateBackendEnum getStateBackend() {
				return backendEnum;
			}

			@Override
			protected Configuration createClusterConfig() throws IOException {
				Configuration config = super.createClusterConfig();

				config.setBoolean(
					CheckpointingOptions.LOCAL_RECOVERY,
					localRecoveryEnabled);

				return config;
			}
		};

	executeTest(windowChkITCase);
}
 

@Test
public void testSettingIllegalMaxNumberOfCheckpointsToRetain() throws Exception {

	final int negativeMaxNumberOfCheckpointsToRetain = -10;

	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		negativeMaxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertNotEquals(negativeMaxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 

@Override
public FsStateBackend createFromConfig(Configuration config, ClassLoader classLoader) throws IllegalConfigurationException {
	// we need to explicitly read the checkpoint directory here, because that
	// is a required constructor parameter
	final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY);
	if (checkpointDir == null) {
		throw new IllegalConfigurationException(
				"Cannot create the file system state backend: The configuration does not specify the " +
						"checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\'');
	}

	try {
		return new FsStateBackend(checkpointDir).configure(config, classLoader);
	}
	catch (IllegalArgumentException e) {
		throw new IllegalConfigurationException("Invalid configuration for the state backend", e);
	}
}
 

/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testConfigureMemoryStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointPath = new Path(appCheckpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final MemoryStateBackend backend = new MemoryStateBackend(appCheckpointDir, null);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this parameter should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
}
 

/**
 * Private constructor that creates a re-configured copy of the state backend.
 *
 * @param original The state backend to re-configure
 * @param configuration The configuration
 * @param classLoader The class loader
 */
private MemoryStateBackend(MemoryStateBackend original, Configuration configuration, ClassLoader classLoader) {
	super(original.getCheckpointPath(), original.getSavepointPath(), configuration);

	this.maxStateSize = original.maxStateSize;

	// if asynchronous snapshots were configured, use that setting,
	// else check the configuration
	this.asynchronousSnapshots = original.asynchronousSnapshots.resolveUndefined(
			configuration.getBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS));
}
 

/**
 * Tests that if local recovery is disabled we won't spread
 * out tasks when recovering.
 */
@Test
public void testDisablingLocalRecovery() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, false);

	executeSchedulingTest(configuration);
}
 

/**
 * Validates loading a memory state backend with additional parameters from the cluster configuration.
 */
@Test
public void testLoadMemoryStateWithParameters() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointPath = new Path(checkpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	// we configure with the explicit string (rather than AbstractStateBackend#X_STATE_BACKEND_NAME)
	// to guard against config-breaking changes of the name

	final Configuration config1 = new Configuration();
	config1.setString(backendKey, "jobmanager");
	config1.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config1.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config1.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	final Configuration config2 = new Configuration();
	config2.setString(backendKey, MemoryStateBackendFactory.class.getName());
	config2.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config2.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config2.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, async);

	MemoryStateBackend backend1 = (MemoryStateBackend)
			StateBackendLoader.loadStateBackendFromConfig(config1, cl, null);
	MemoryStateBackend backend2 = (MemoryStateBackend)
			StateBackendLoader.loadStateBackendFromConfig(config2, cl, null);

	assertNotNull(backend1);
	assertNotNull(backend2);

	assertEquals(expectedCheckpointPath, backend1.getCheckpointPath());
	assertEquals(expectedCheckpointPath, backend2.getCheckpointPath());
	assertEquals(expectedSavepointPath, backend1.getSavepointPath());
	assertEquals(expectedSavepointPath, backend2.getSavepointPath());
	assertEquals(async, backend1.isUsingAsynchronousSnapshots());
	assertEquals(async, backend2.isUsingAsynchronousSnapshots());
}
 

@Override
public CompletableFuture<String> triggerSavepoint(final String targetDirectory, final boolean cancelJob) {
	mainThreadExecutor.assertRunningInMainThread();

	final CheckpointCoordinator checkpointCoordinator = executionGraph.getCheckpointCoordinator();
	if (checkpointCoordinator == null) {
		throw new IllegalStateException(
			String.format("Job %s is not a streaming job.", jobGraph.getJobID()));
	} else if (targetDirectory == null && !checkpointCoordinator.getCheckpointStorage().hasDefaultSavepointLocation()) {
		log.info("Trying to cancel job {} with savepoint, but no savepoint directory configured.", jobGraph.getJobID());

		throw new IllegalStateException(
			"No savepoint directory configured. You can either specify a directory " +
				"while cancelling via -s :targetDirectory or configure a cluster-wide " +
				"default via key '" + CheckpointingOptions.SAVEPOINT_DIRECTORY.key() + "'.");
	}

	if (cancelJob) {
		checkpointCoordinator.stopCheckpointScheduler();
	}

	return checkpointCoordinator
		.triggerSavepoint(System.currentTimeMillis(), targetDirectory)
		.thenApply(CompletedCheckpoint::getExternalPointer)
		.handleAsync((path, throwable) -> {
			if (throwable != null) {
				if (cancelJob) {
					startCheckpointScheduler(checkpointCoordinator);
				}
				throw new CompletionException(throwable);
			} else if (cancelJob) {
				log.info("Savepoint stored in {}. Now cancelling {}.", path, jobGraph.getJobID());
				cancel();
			}
			return path;
		}, mainThreadExecutor);
}
 

@Nonnull
private static SlotSelectionStrategy selectSlotSelectionStrategy(@Nonnull Configuration configuration) {
	if (configuration.getBoolean(CheckpointingOptions.LOCAL_RECOVERY)) {
		return PreviousAllocationSlotSelectionStrategy.INSTANCE;
	} else {
		return LocationPreferenceSlotSelectionStrategy.INSTANCE;
	}
}
 

@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 

@Test
public void testDefaultsInSync() throws Exception {
	final boolean defaultIncremental = CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	RocksDBStateBackend backend = new RocksDBStateBackend(tempFolder.newFolder().toURI());
	assertEquals(defaultIncremental, backend.isIncrementalCheckpointsEnabled());
}
 

private Configuration getFileBasedCheckpointsConfig(final String savepointDir) {
	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 0);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	return config;
}
 

/**
 * Validates taking the application-defined memory state backend and adding additional
 * parameters from the cluster configuration.
 */
@Test
public void testConfigureMemoryStateBackend() throws Exception {
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();
	final Path expectedCheckpointPath = new Path(checkpointDir);
	final Path expectedSavepointPath = new Path(savepointDir);

	final int maxSize = 100;
	final boolean async = !CheckpointingOptions.ASYNC_SNAPSHOTS.defaultValue();

	final MemoryStateBackend backend = new MemoryStateBackend(maxSize, async);

	final Configuration config = new Configuration();
	config.setString(backendKey, "filesystem"); // check that this is not accidentally picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir);
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS, !async);

	StateBackend loadedBackend = StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof MemoryStateBackend);

	final MemoryStateBackend memBackend = (MemoryStateBackend) loadedBackend;
	assertEquals(expectedCheckpointPath, memBackend.getCheckpointPath());
	assertEquals(expectedSavepointPath, memBackend.getSavepointPath());
	assertEquals(maxSize, memBackend.getMaxStateSize());
	assertEquals(async, memBackend.isUsingAsynchronousSnapshots());
}
 

/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final String localDir1 = tmp.newFolder().getAbsolutePath();
	final String localDir2 = tmp.newFolder().getAbsolutePath();
	final String localDir3 = tmp.newFolder().getAbsolutePath();
	final String localDir4 = tmp.newFolder().getAbsolutePath();

	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final Path expectedCheckpointsPath = new Path(appCheckpointDir);
	final Path expectedSavepointsPath = new Path(savepointDir);

	final RocksDBStateBackend backend = new RocksDBStateBackend(appCheckpointDir, incremental);
	backend.setDbStoragePaths(localDir1, localDir2);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setBoolean(CheckpointingOptions.INCREMENTAL_CHECKPOINTS, !incremental);  // this should not be picked up
	config.setString(RocksDBOptions.LOCAL_DIRECTORIES, localDir3 + ":" + localDir4);  // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof RocksDBStateBackend);

	final RocksDBStateBackend loadedRocks = (RocksDBStateBackend) loadedBackend;

	assertEquals(incremental, loadedRocks.isIncrementalCheckpointsEnabled());
	checkPaths(loadedRocks.getDbStoragePaths(), localDir1, localDir2);

	AbstractFileStateBackend fsBackend = (AbstractFileStateBackend) loadedRocks.getCheckpointBackend();
	assertEquals(expectedCheckpointsPath, fsBackend.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fsBackend.getSavepointPath());
}
 

@Test
public void testStateBackendLoadingAndClosing() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = false;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

@Test
public void testStateBackendLoadingAndClosing() throws Exception {
	Configuration taskManagerConfig = new Configuration();
	taskManagerConfig.setString(CheckpointingOptions.STATE_BACKEND, TestMemoryStateBackendFactory.class.getName());

	StreamConfig cfg = new StreamConfig(new Configuration());
	cfg.setStateKeySerializer(mock(TypeSerializer.class));
	cfg.setOperatorID(new OperatorID(4711L, 42L));
	TestStreamSource<Long, MockSourceFunction> streamSource = new TestStreamSource<>(new MockSourceFunction());
	cfg.setStreamOperator(streamSource);
	cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);

	Task task = createTask(StateBackendTestSource.class, cfg, taskManagerConfig);

	StateBackendTestSource.fail = false;
	task.startTaskThread();

	// wait for clean termination
	task.getExecutingThread().join();

	// ensure that the state backends and stream iterables are closed ...
	verify(TestStreamSource.operatorStateBackend).close();
	verify(TestStreamSource.keyedStateBackend).close();
	verify(TestStreamSource.rawOperatorStateInputs).close();
	verify(TestStreamSource.rawKeyedStateInputs).close();
	// ... and disposed
	verify(TestStreamSource.operatorStateBackend).dispose();
	verify(TestStreamSource.keyedStateBackend).dispose();

	assertEquals(ExecutionState.FINISHED, task.getExecutionState());
}
 

/**
 * Sets all necessary configuration keys to operate in {@link HighAvailabilityMode#ZOOKEEPER}.
 *
 * @param config            Configuration to use
 * @param zooKeeperQuorum   ZooKeeper quorum to connect to
 * @param fsStateHandlePath Base path for file system state backend (for checkpoints and
 *                          recovery)
 * @return The modified configuration to operate in {@link HighAvailabilityMode#ZOOKEEPER}.
 */
public static Configuration configureZooKeeperHA(
		Configuration config,
		String zooKeeperQuorum,
		String fsStateHandlePath) {

	checkNotNull(config, "Configuration");
	checkNotNull(zooKeeperQuorum, "ZooKeeper quorum");
	checkNotNull(fsStateHandlePath, "File state handle backend path");

	// ZooKeeper recovery mode
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperQuorum);

	int connTimeout = 5000;
	if (System.getenv().containsKey("CI")) {
		// The regular timeout is to aggressive for Travis and connections are often lost.
		connTimeout = 30000;
	}

	config.setInteger(HighAvailabilityOptions.ZOOKEEPER_CONNECTION_TIMEOUT, connTimeout);
	config.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, connTimeout);

	// File system state backend
	config.setString(CheckpointingOptions.STATE_BACKEND, "FILESYSTEM");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, fsStateHandlePath + "/checkpoints");
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, fsStateHandlePath + "/recovery");

	// Akka failure detection and execution retries
	config.setString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL, "1000 ms");
	config.setString(AkkaOptions.WATCH_HEARTBEAT_PAUSE, "6 s");
	config.setInteger(AkkaOptions.WATCH_THRESHOLD, 9);
	config.setString(AkkaOptions.ASK_TIMEOUT, "100 s");
	config.setString(HighAvailabilityOptions.HA_JOB_DELAY, "10 s");

	return config;
}
 

@Test
public void testSettingMaxNumberOfCheckpointsToRetain() throws Exception {

	final int maxNumberOfCheckpointsToRetain = 10;
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setInteger(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS,
		maxNumberOfCheckpointsToRetain);

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertEquals(maxNumberOfCheckpointsToRetain,
		eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 

@Test
public void testSettingDefaultMaxNumberOfCheckpointsToRetain() throws Exception {
	final Configuration jobManagerConfig = new Configuration();

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}
 

/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), threshold);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
}
 

@Test
public void testSettingDefaultMaxNumberOfCheckpointsToRetain() throws Exception {
	final Configuration jobManagerConfig = new Configuration();

	final ExecutionGraph eg = createExecutionGraph(jobManagerConfig);

	assertEquals(CheckpointingOptions.MAX_RETAINED_CHECKPOINTS.defaultValue().intValue(),
			eg.getCheckpointCoordinator().getCheckpointStore().getMaxNumberOfRetainedCheckpoints());
}