Java Code Examples for org.apache.flink.runtime.state.filesystem.FsStateBackend

The following examples show how to use org.apache.flink.runtime.state.filesystem.FsStateBackend. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: RocksDBTtlStateTestBase.java    License: Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 3
Source Project: Flink-CEPplus   Source File: StatefulStreamingJob.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example 4
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(1000);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

    CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
            .setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
            .enableWriteAheadLog()
            .setClusterBuilder(new ClusterBuilder() {

                private static final long serialVersionUID = 2793938419775311824L;

                @Override
                public Cluster buildCluster(Cluster.Builder builder) {
                    return builder.addContactPoint("127.0.0.1").build();
                }
            })
            .build();

    sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

    env.execute();
}
 
Example 5
Source Project: flink   Source File: CassandraTupleWriteAheadSinkExample.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example 6
Source Project: flink   Source File: RocksDBTtlStateTestBase.java    License: Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 7
Source Project: flink   Source File: DataStreamAllroundTestJobFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 
Example 8
Source Project: flink   Source File: StatefulStreamingJob.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example 9
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(1000);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

    CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
            .setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
            .enableWriteAheadLog()
            .setClusterBuilder(new ClusterBuilder() {

                private static final long serialVersionUID = 2793938419775311824L;

                @Override
                public Cluster buildCluster(Cluster.Builder builder) {
                    return builder.addContactPoint("127.0.0.1").build();
                }
            })
            .build();

    sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

    env.execute();
}
 
Example 10
/**
 * Do some Flink Configuration
 */
private static StreamExecutionEnvironment setupFlinkEnvironment(BenchmarkConfig config) throws IOException {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().setGlobalJobParameters(config.getParameters());
  env.getConfig().enableObjectReuse();

  // enable checkpointing for fault tolerance
  if (config.checkpointsEnabled) {
    env.enableCheckpointing(config.checkpointInterval);
    if (config.checkpointToUri) {
      env.setStateBackend(new FsStateBackend(config.checkpointUri));
    }
  }

  // use event time
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  return env;
}
 
Example 11
Source Project: flink   Source File: CassandraTupleWriteAheadSinkExample.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example 12
Source Project: flink   Source File: EventTimeWindowCheckpointingITCase.java    License: Apache License 2.0 6 votes vote down vote up
private void setupRocksDB(Configuration config, int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	// Configure the managed memory size as 64MB per slot for rocksDB state backend.
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.ofMebiBytes(PARALLELISM / NUM_OF_TASK_MANAGERS * 64));

	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example 13
Source Project: flink   Source File: DataStreamAllroundTestJobFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 
Example 14
Source Project: flink   Source File: StatefulStreamingJob.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example 15
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example 16
Source Project: Flink-CEPplus   Source File: RocksDBStateBackendTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 17
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 18
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 
Example 19
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
	FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
	assertTrue(backend.isUsingAsynchronousSnapshots());

	validateSupportForAsyncSnapshots(backend);
}
 
Example 20
Source Project: Flink-CEPplus   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), threshold);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
}
 
Example 21
Source Project: flink   Source File: EventTimeWindowCheckpointingITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example 22
Source Project: flink   Source File: RocksDBStateBackendTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 23
Source Project: flink   Source File: RocksDBStateBackendMigrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 24
Source Project: flink   Source File: RocksDBStateBackendConfigTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 
Example 25
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
	FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
	assertTrue(backend.isUsingAsynchronousSnapshots());

	validateSupportForAsyncSnapshots(backend);
}
 
Example 26
Source Project: flink   Source File: StateBackendLoadingTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example 27
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", ExerciseBase.pathToRideData);
		final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(ExerciseBase.parallelism);

		// set up checkpointing
		env.setStateBackend(new FsStateBackend("file:///tmp/checkpoints"));
		env.enableCheckpointing(1000);
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(60, Time.of(10, TimeUnit.SECONDS)));

		DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(input, servingSpeedFactor)));

		DataStream<TaxiRide> longRides = rides
				.filter(new NYCFilter())
				.keyBy((TaxiRide ride) -> ride.rideId)
				.process(new MatchFunction());

		printOrTest(longRides);

		env.execute("Long Taxi Rides (checkpointed)");
	}
 
Example 28
Source Project: flink   Source File: RocksDBTtlStateTestBase.java    License: Apache License 2.0 5 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 29
Source Project: flink   Source File: RocksDBStateBackendTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration configuration = new Configuration();
	configuration.set(RocksDBOptions.TIMER_SERVICE_FACTORY, RocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example 30
Source Project: flink   Source File: RocksDBStateBackendMigrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.set(RocksDBOptions.TIMER_SERVICE_FACTORY, RocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}