org.apache.flink.runtime.state.filesystem.FsStateBackend Java Examples

The following examples show how to use org.apache.flink.runtime.state.filesystem.FsStateBackend. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StatefulStreamingJob.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example #2
Source File: RocksDBTtlStateTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #3
Source File: DataStreamAllroundTestJobFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 
Example #4
Source File: StatefulStreamingJob.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example #5
Source File: StatefulStreamingJob.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool pt = ParameterTool.fromArgs(args);
	final String checkpointDir = pt.getRequired("checkpoint.dir");

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(new FsStateBackend(checkpointDir));
	env.setRestartStrategy(RestartStrategies.noRestart());
	env.enableCheckpointing(1000L);
	env.getConfig().disableGenericTypes();

	env.addSource(new MySource()).uid("my-source")
			.keyBy(anInt -> 0)
			.map(new MyStatefulFunction()).uid("my-map")
			.addSink(new DiscardingSink<>()).uid("my-sink");
	env.execute();
}
 
Example #6
Source File: EventTimeWindowCheckpointingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private void setupRocksDB(Configuration config, int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	// Configure the managed memory size as 64MB per slot for rocksDB state backend.
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.ofMebiBytes(PARALLELISM / NUM_OF_TASK_MANAGERS * 64));

	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example #7
Source File: CassandraTupleWriteAheadSinkExample.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(1000);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

    CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
            .setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
            .enableWriteAheadLog()
            .setClusterBuilder(new ClusterBuilder() {

                private static final long serialVersionUID = 2793938419775311824L;

                @Override
                public Cluster buildCluster(Cluster.Builder builder) {
                    return builder.addContactPoint("127.0.0.1").build();
                }
            })
            .build();

    sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

    env.execute();
}
 
Example #8
Source File: CassandraTupleWriteAheadSinkExample.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example #9
Source File: CassandraTupleWriteAheadSinkExample.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example #10
Source File: AdvertisingTopologyFlinkStateHighKeyCard.java    From yahoo-streaming-benchmark with Apache License 2.0 6 votes vote down vote up
/**
 * Do some Flink Configuration
 */
private static StreamExecutionEnvironment setupFlinkEnvironment(BenchmarkConfig config) throws IOException {
  StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
  env.getConfig().setGlobalJobParameters(config.getParameters());
  env.getConfig().enableObjectReuse();

  // enable checkpointing for fault tolerance
  if (config.checkpointsEnabled) {
    env.enableCheckpointing(config.checkpointInterval);
    if (config.checkpointToUri) {
      env.setStateBackend(new FsStateBackend(config.checkpointUri));
    }
  }

  // use event time
  env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
  return env;
}
 
Example #11
Source File: RocksDBTtlStateTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #12
Source File: CassandraTupleWriteAheadSinkExample.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(1000);
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
	env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

	CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
		.setQuery("INSERT INTO example.values (id, counter) values (?, ?);")
		.enableWriteAheadLog()
		.setClusterBuilder(new ClusterBuilder() {

			private static final long serialVersionUID = 2793938419775311824L;

			@Override
			public Cluster buildCluster(Cluster.Builder builder) {
				return builder.addContactPoint("127.0.0.1").build();
			}
		})
		.build();

	sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

	env.execute();
}
 
Example #13
Source File: CassandraTupleWriteAheadSinkExample.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    env.enableCheckpointing(1000);
    env.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 1000));
    env.setStateBackend(new FsStateBackend("file:///" + System.getProperty("java.io.tmpdir") + "/flink/backend"));

    CassandraSink<Tuple2<String, Integer>> sink = CassandraSink.addSink(env.addSource(new MySource()))
            .setQuery("INSERT INTO zhisheng.values (id, counter) values (?, ?);")
            .enableWriteAheadLog()
            .setClusterBuilder(new ClusterBuilder() {

                private static final long serialVersionUID = 2793938419775311824L;

                @Override
                public Cluster buildCluster(Cluster.Builder builder) {
                    return builder.addContactPoint("127.0.0.1").build();
                }
            })
            .build();

    sink.name("Cassandra Sink").disableChaining().setParallelism(1).uid("hello");

    env.execute();
}
 
Example #14
Source File: DataStreamAllroundTestJobFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void setupStateBackend(final StreamExecutionEnvironment env, final ParameterTool pt) throws IOException {
	final String stateBackend = pt.get(
		STATE_BACKEND.key(),
		STATE_BACKEND.defaultValue());

	final String checkpointDir = pt.getRequired(STATE_BACKEND_CHECKPOINT_DIR.key());

	if ("file".equalsIgnoreCase(stateBackend)) {
		boolean asyncCheckpoints = pt.getBoolean(
			STATE_BACKEND_FILE_ASYNC.key(),
			STATE_BACKEND_FILE_ASYNC.defaultValue());

		env.setStateBackend((StateBackend) new FsStateBackend(checkpointDir, asyncCheckpoints));
	} else if ("rocks".equalsIgnoreCase(stateBackend)) {
		boolean incrementalCheckpoints = pt.getBoolean(
			STATE_BACKEND_ROCKS_INCREMENTAL.key(),
			STATE_BACKEND_ROCKS_INCREMENTAL.defaultValue());

		env.setStateBackend((StateBackend) new RocksDBStateBackend(checkpointDir, incrementalCheckpoints));
	} else {
		throw new IllegalArgumentException("Unknown backend requested: " + stateBackend);
	}
}
 
Example #15
Source File: RocksDBStateBackendMigrationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.set(RocksDBOptions.TIMER_SERVICE_FACTORY, RocksDBStateBackend.PriorityQueueStateType.ROCKSDB);
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #16
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
	FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
	assertTrue(backend.isUsingAsynchronousSnapshots());

	validateSupportForAsyncSnapshots(backend);
}
 
Example #17
Source File: RocksDBStateBackendTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #18
Source File: RocksDBStateBackendMigrationTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #19
Source File: RocksDBTtlStateTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #20
Source File: RocksDBStateBackendConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final RocksDBOptionsFactory optionsFactory = mock(RocksDBOptionsFactory.class);
	original.setRocksDBOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getRocksDBOptions(), copy.getRocksDBOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 
Example #21
Source File: EventTimeWindowCheckpointingITCase.java    From flink with Apache License 2.0 5 votes vote down vote up
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example #22
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
	FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
	assertTrue(backend.isUsingAsynchronousSnapshots());

	validateSupportForAsyncSnapshots(backend);
}
 
Example #23
Source File: CheckpointedLongRidesSolution.java    From flink-training-exercises with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", ExerciseBase.pathToRideData);
		final int servingSpeedFactor = 1800; // 30 minutes worth of events are served every second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
		env.setParallelism(ExerciseBase.parallelism);

		// set up checkpointing
		env.setStateBackend(new FsStateBackend("file:///tmp/checkpoints"));
		env.enableCheckpointing(1000);
		env.setRestartStrategy(RestartStrategies.fixedDelayRestart(60, Time.of(10, TimeUnit.SECONDS)));

		DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new CheckpointedTaxiRideSource(input, servingSpeedFactor)));

		DataStream<TaxiRide> longRides = rides
				.filter(new NYCFilter())
				.keyBy((TaxiRide ride) -> ride.rideId)
				.process(new MatchFunction());

		printOrTest(longRides);

		env.execute("Long Taxi Rides (checkpointed)");
	}
 
Example #24
Source File: RocksDBStateBackendTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #25
Source File: RocksDBStateBackendMigrationTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected RocksDBStateBackend getStateBackend() throws IOException {
	dbPath = tempFolder.newFolder().getAbsolutePath();
	String checkpointPath = tempFolder.newFolder().toURI().toString();
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);

	Configuration configuration = new Configuration();
	configuration.setString(
		RocksDBOptions.TIMER_SERVICE_FACTORY,
		RocksDBStateBackend.PriorityQueueStateType.ROCKSDB.toString());
	backend = backend.configure(configuration, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #26
Source File: RocksDBStateBackendConfigTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}
 
Example #27
Source File: EventTimeWindowCheckpointingITCase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException {
	String rocksDb = tempFolder.newFolder().getAbsolutePath();
	String backups = tempFolder.newFolder().getAbsolutePath();
	// we use the fs backend with small threshold here to test the behaviour with file
	// references, not self contained byte handles
	RocksDBStateBackend rdb =
		new RocksDBStateBackend(
			new FsStateBackend(
				new Path("file://" + backups).toUri(), fileSizeThreshold),
			incrementalCheckpoints);
	rdb.setDbStoragePath(rocksDb);
	this.stateBackend = rdb;
}
 
Example #28
Source File: HeapKeyedStateBackendAsyncByDefaultTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFsStateBackendDefaultsToAsync() throws Exception {
	FsStateBackend backend = new FsStateBackend(tmpFolder.newFolder().toURI());
	assertTrue(backend.isUsingAsynchronousSnapshots());

	validateSupportForAsyncSnapshots(backend);
}
 
Example #29
Source File: StateBackendLoadingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example #30
Source File: RocksDBStateBackendConfigTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testRocksDbReconfigurationCopiesExistingValues() throws Exception {
	final FsStateBackend checkpointBackend = new FsStateBackend(tempFolder.newFolder().toURI().toString());
	final boolean incremental = !CheckpointingOptions.INCREMENTAL_CHECKPOINTS.defaultValue();

	final RocksDBStateBackend original = new RocksDBStateBackend(checkpointBackend, incremental);

	// these must not be the default options
	final PredefinedOptions predOptions = PredefinedOptions.SPINNING_DISK_OPTIMIZED_HIGH_MEM;
	assertNotEquals(predOptions, original.getPredefinedOptions());
	original.setPredefinedOptions(predOptions);

	final OptionsFactory optionsFactory = mock(OptionsFactory.class);
	original.setOptions(optionsFactory);

	final String[] localDirs = new String[] {
			tempFolder.newFolder().getAbsolutePath(), tempFolder.newFolder().getAbsolutePath() };
	original.setDbStoragePaths(localDirs);

	RocksDBStateBackend copy = original.configure(new Configuration(), Thread.currentThread().getContextClassLoader());

	assertEquals(original.isIncrementalCheckpointsEnabled(), copy.isIncrementalCheckpointsEnabled());
	assertArrayEquals(original.getDbStoragePaths(), copy.getDbStoragePaths());
	assertEquals(original.getOptions(), copy.getOptions());
	assertEquals(original.getPredefinedOptions(), copy.getPredefinedOptions());

	FsStateBackend copyCheckpointBackend = (FsStateBackend) copy.getCheckpointBackend();
	assertEquals(checkpointBackend.getCheckpointPath(), copyCheckpointBackend.getCheckpointPath());
	assertEquals(checkpointBackend.getSavepointPath(), copyCheckpointBackend.getSavepointPath());
}