org.apache.flink.util.TernaryBoolean Java Examples

The following examples show how to use org.apache.flink.util.TernaryBoolean. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBTtlStateTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #2
Source File: RocksDBTtlStateTestBase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	config.setBoolean(TTL_COMPACT_FILTER_ENABLED, true);
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #3
Source File: CsvTableSinkFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private DescriptorProperties createDescriptor(TableSchema schema) {
	Map<String, String> properties = new HashMap<>();
	properties.put("connector.type", "filesystem");
	properties.put("connector.property-version", "1");
	properties.put("connector.path", "/path/to/csv");

	// schema
	properties.put("format.type", "csv");
	properties.put("format.property-version", "1");
	properties.put("format.field-delimiter", ";");

	DescriptorProperties descriptor = new DescriptorProperties(true);
	descriptor.putProperties(properties);
	descriptor.putTableSchema(SCHEMA, schema);
	if (deriveSchema == TernaryBoolean.TRUE) {
		descriptor.putBoolean("format.derive-schema", true);
	} else if (deriveSchema == TernaryBoolean.FALSE) {
		descriptor.putTableSchema(FORMAT_FIELDS, testingSchema);
	} // nothing to put for UNDEFINED
	return descriptor;
}
 
Example #4
Source File: StateBackendLoadingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.set(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, MemorySize.parse("20")); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example #5
Source File: StateBackendLoadingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Validates taking the application-defined file system state backend and adding with additional
 * parameters from the cluster configuration, but giving precedence to application-defined
 * parameters over configuration-defined parameters.
 */
@Test
public void testLoadFileSystemStateBackendMixed() throws Exception {
	final String appCheckpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String checkpointDir = new Path(tmp.newFolder().toURI()).toString();
	final String savepointDir = new Path(tmp.newFolder().toURI()).toString();

	final Path expectedCheckpointsPath = new Path(new URI(appCheckpointDir));
	final Path expectedSavepointsPath = new Path(savepointDir);

	final int threshold = 1000000;
	final int writeBufferSize = 4000000;

	final FsStateBackend backend = new FsStateBackend(new URI(appCheckpointDir), null, threshold, writeBufferSize, TernaryBoolean.TRUE);

	final Configuration config = new Configuration();
	config.setString(backendKey, "jobmanager"); // this should not be picked up 
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir); // this should not be picked up
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir);
	config.setInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD, 20); // this should not be picked up
	config.setInteger(CheckpointingOptions.FS_WRITE_BUFFER_SIZE, 3000000); // this should not be picked up

	final StateBackend loadedBackend =
			StateBackendLoader.fromApplicationOrConfigOrDefault(backend, config, cl, null);
	assertTrue(loadedBackend instanceof FsStateBackend);

	final FsStateBackend fs = (FsStateBackend) loadedBackend;
	assertEquals(expectedCheckpointsPath, fs.getCheckpointPath());
	assertEquals(expectedSavepointsPath, fs.getSavepointPath());
	assertEquals(threshold, fs.getMinFileSizeThreshold());
	assertEquals(writeBufferSize, fs.getWriteBufferSize());
}
 
Example #6
Source File: RocksDBTtlStateTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
StateBackend createStateBackend(TernaryBoolean enableIncrementalCheckpointing) {
	String dbPath;
	String checkpointPath;
	try {
		dbPath = tempFolder.newFolder().getAbsolutePath();
		checkpointPath = tempFolder.newFolder().toURI().toString();
	} catch (IOException e) {
		throw new FlinkRuntimeException("Failed to init rocksdb test state backend");
	}
	RocksDBStateBackend backend = new RocksDBStateBackend(new FsStateBackend(checkpointPath), enableIncrementalCheckpointing);
	Configuration config = new Configuration();
	backend = backend.configure(config, Thread.currentThread().getContextClassLoader());
	backend.setDbStoragePath(dbPath);
	return backend;
}
 
Example #7
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) {
	this(checkpointStreamBackend, TernaryBoolean.UNDEFINED);
}
 
Example #8
Source File: IncSnapshotRocksDbTtlStateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.TRUE);
}
 
Example #9
Source File: FullSnapshotRocksDbTtlStateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.FALSE);
}
 
Example #10
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) {
	this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing));
}
 
Example #11
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) {
	this(checkpointStreamBackend, TernaryBoolean.UNDEFINED);
}
 
Example #12
Source File: FsStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDirectory        The path to write checkpoint metadata to.
 * @param defaultSavepointDirectory  The path to write savepoints to. If null, the value from
 *                                   the runtime configuration will be used, or savepoint
 *                                   target locations need to be passed when triggering a savepoint.
 * @param fileStateSizeThreshold     State below this size will be stored as part of the metadata,
 *                                   rather than in files. If -1, the value configured in the
 *                                   runtime configuration will be used, or the default value (1KB)
 *                                   if nothing is configured.
 * @param writeBufferSize            Write buffer size used to serialize state. If -1, the value configured in the
 *                                   runtime configuration will be used, or the default value (4KB)
 *                                   if nothing is configured.
 * @param asynchronousSnapshots      Flag to switch between synchronous and asynchronous
 *                                   snapshot mode. If UNDEFINED, the value configured in the
 *                                   runtime configuration will be used.
 */
public FsStateBackend(
		URI checkpointDirectory,
		@Nullable URI defaultSavepointDirectory,
		int fileStateSizeThreshold,
		int writeBufferSize,
		TernaryBoolean asynchronousSnapshots) {

	super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory);

	checkNotNull(asynchronousSnapshots, "asynchronousSnapshots");
	checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD,
			"The threshold for file state size must be in [-1, %s], where '-1' means to use " +
					"the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD);
	checkArgument(writeBufferSize >= -1 ,
		"The write buffer size must be not less than '-1', where '-1' means to use " +
			"the value from the deployment's configuration.");

	this.fileStateThreshold = fileStateSizeThreshold;
	this.writeBufferSize = writeBufferSize;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
 
Example #13
Source File: CsvTableSinkFactoryTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Parameterized.Parameters(name = "deriveSchema = {0}")
public static TernaryBoolean[] getDeriveSchema() {
	return new TernaryBoolean[]{TernaryBoolean.TRUE, TernaryBoolean.FALSE, TernaryBoolean.UNDEFINED};
}
 
Example #14
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) {
	this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing));
}
 
Example #15
Source File: FsStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDirectory        The path to write checkpoint metadata to.
 * @param defaultSavepointDirectory  The path to write savepoints to. If null, the value from
 *                                   the runtime configuration will be used, or savepoint
 *                                   target locations need to be passed when triggering a savepoint.
 * @param fileStateSizeThreshold     State below this size will be stored as part of the metadata,
 *                                   rather than in files. If -1, the value configured in the
 *                                   runtime configuration will be used, or the default value (1KB)
 *                                   if nothing is configured.
 * @param asynchronousSnapshots      Flag to switch between synchronous and asynchronous
 *                                   snapshot mode. If UNDEFINED, the value configured in the
 *                                   runtime configuration will be used.
 */
public FsStateBackend(
		URI checkpointDirectory,
		@Nullable URI defaultSavepointDirectory,
		int fileStateSizeThreshold,
		TernaryBoolean asynchronousSnapshots) {

	super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory);

	checkNotNull(asynchronousSnapshots, "asynchronousSnapshots");
	checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD,
			"The threshold for file state size must be in [-1, %s], where '-1' means to use " +
					"the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD);

	this.fileStateThreshold = fileStateSizeThreshold;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
 
Example #16
Source File: FullSnapshotRocksDbTtlStateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.FALSE);
}
 
Example #17
Source File: IncSnapshotRocksDbTtlStateTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.TRUE);
}
 
Example #18
Source File: FsStateBackend.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDirectory        The path to write checkpoint metadata to.
 * @param defaultSavepointDirectory  The path to write savepoints to. If null, the value from
 *                                   the runtime configuration will be used, or savepoint
 *                                   target locations need to be passed when triggering a savepoint.
 * @param fileStateSizeThreshold     State below this size will be stored as part of the metadata,
 *                                   rather than in files. If -1, the value configured in the
 *                                   runtime configuration will be used, or the default value (1KB)
 *                                   if nothing is configured.
 * @param writeBufferSize            Write buffer size used to serialize state. If -1, the value configured in the
 *                                   runtime configuration will be used, or the default value (4KB)
 *                                   if nothing is configured.
 * @param asynchronousSnapshots      Flag to switch between synchronous and asynchronous
 *                                   snapshot mode. If UNDEFINED, the value configured in the
 *                                   runtime configuration will be used.
 */
public FsStateBackend(
		URI checkpointDirectory,
		@Nullable URI defaultSavepointDirectory,
		int fileStateSizeThreshold,
		int writeBufferSize,
		TernaryBoolean asynchronousSnapshots) {

	super(checkNotNull(checkpointDirectory, "checkpoint directory is null"), defaultSavepointDirectory);

	checkNotNull(asynchronousSnapshots, "asynchronousSnapshots");
	checkArgument(fileStateSizeThreshold >= -1 && fileStateSizeThreshold <= MAX_FILE_STATE_THRESHOLD,
			"The threshold for file state size must be in [-1, %s], where '-1' means to use " +
					"the value from the deployment's configuration.", MAX_FILE_STATE_THRESHOLD);
	checkArgument(writeBufferSize >= -1 ,
		"The write buffer size must be not less than '-1', where '-1' means to use " +
			"the value from the deployment's configuration.");

	this.fileStateThreshold = fileStateSizeThreshold;
	this.writeBufferSize = writeBufferSize;
	this.asynchronousSnapshots = asynchronousSnapshots;
}
 
Example #19
Source File: IncSnapshotRocksDbTtlStateTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.TRUE);
}
 
Example #20
Source File: FullSnapshotRocksDbTtlStateTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
StateBackend createStateBackend() {
	return createStateBackend(TernaryBoolean.FALSE);
}
 
Example #21
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend, TernaryBoolean)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend, boolean enableIncrementalCheckpointing) {
	this(checkpointStreamBackend, TernaryBoolean.fromBoolean(enableIncrementalCheckpointing));
}
 
Example #22
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * @deprecated Use {@link #RocksDBStateBackend(StateBackend)} instead.
 */
@Deprecated
public RocksDBStateBackend(AbstractStateBackend checkpointStreamBackend) {
	this(checkpointStreamBackend, TernaryBoolean.UNDEFINED);
}
 
Example #23
Source File: RocksDBStateBackend.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its
 * checkpoint data streams. Typically, one would supply a filesystem or database state backend
 * here where the snapshots from RocksDB would be stored.
 *
 * <p>The snapshots of the RocksDB state will be stored using the given backend's
 * {@link StateBackend#createCheckpointStorage(JobID)}.
 *
 * @param checkpointStreamBackend The backend write the checkpoint streams to.
 * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled.
 */
public RocksDBStateBackend(StateBackend checkpointStreamBackend, TernaryBoolean enableIncrementalCheckpointing) {
	this.checkpointStreamBackend = checkNotNull(checkpointStreamBackend);
	this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
	this.numberOfTransferingThreads = UNDEFINED_NUMBER_OF_TRANSFERING_THREADS;
	// for now, we use still the heap-based implementation as default
	this.priorityQueueStateType = PriorityQueueStateType.HEAP;
	this.defaultMetricOptions = new RocksDBNativeMetricOptions();
	this.enableTtlCompactionFilter = TernaryBoolean.UNDEFINED;
}
 
Example #24
Source File: RocksDBStateBackend.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new {@code RocksDBStateBackend} that uses the given state backend to store its
 * checkpoint data streams. Typically, one would supply a filesystem or database state backend
 * here where the snapshots from RocksDB would be stored.
 *
 * <p>The snapshots of the RocksDB state will be stored using the given backend's
 * {@link StateBackend#createCheckpointStorage(JobID)}.
 *
 * @param checkpointStreamBackend The backend write the checkpoint streams to.
 * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled.
 */
public RocksDBStateBackend(StateBackend checkpointStreamBackend, TernaryBoolean enableIncrementalCheckpointing) {
	this.checkpointStreamBackend = checkNotNull(checkpointStreamBackend);
	this.enableIncrementalCheckpointing = enableIncrementalCheckpointing;
	this.numberOfTransferThreads = UNDEFINED_NUMBER_OF_TRANSFER_THREADS;
	this.defaultMetricOptions = new RocksDBNativeMetricOptions();
	this.memoryConfiguration = new RocksDBMemoryConfiguration();
	this.writeBatchSize = UNDEFINED_WRITE_BATCH_SIZE;
}
 
Example #25
Source File: FsStateBackend.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata,
 *                             rather than in files (-1 for default value).
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(
		URI checkpointDataUri,
		int fileStateSizeThreshold,
		boolean asynchronousSnapshots) {

	this(checkpointDataUri, null, fileStateSizeThreshold, -1,
			TernaryBoolean.fromBoolean(asynchronousSnapshots));
}
 
Example #26
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata
 * and savepoints to, as well as configuring state thresholds and asynchronous operations.
 *
 * <p><b>WARNING:</b> Increasing the size of this value beyond the default value
 * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care.
 * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there
 * and the JobManager needs to be able to hold all aggregated state in its memory.
 *
 * @param checkpointPath The path to write checkpoint metadata to. If null, the value from
 *                       the runtime configuration will be used.
 * @param savepointPath  The path to write savepoints to. If null, the value from
 *                       the runtime configuration will be used.
 * @param maxStateSize   The maximal size of the serialized state.
 * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous
 *                              snapshot mode. If null, the value configured in the
 *                              runtime configuration will be used.
 */
public MemoryStateBackend(
		@Nullable String checkpointPath,
		@Nullable String savepointPath,
		int maxStateSize,
		TernaryBoolean asynchronousSnapshots) {

	super(checkpointPath == null ? null : new Path(checkpointPath),
			savepointPath == null ? null : new Path(savepointPath));

	checkArgument(maxStateSize > 0, "maxStateSize must be > 0");
	this.maxStateSize = maxStateSize;

	this.asynchronousSnapshots = asynchronousSnapshots;
}
 
Example #27
Source File: FsStateBackend.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata,
 *                             rather than in files (-1 for default value).
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(
		URI checkpointDataUri,
		int fileStateSizeThreshold,
		boolean asynchronousSnapshots) {

	this(checkpointDataUri, null, fileStateSizeThreshold, -1,
			TernaryBoolean.fromBoolean(asynchronousSnapshots));
}
 
Example #28
Source File: MemoryStateBackend.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata
 * and savepoints to, as well as configuring state thresholds and asynchronous operations.
 *
 * <p><b>WARNING:</b> Increasing the size of this value beyond the default value
 * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care.
 * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there
 * and the JobManager needs to be able to hold all aggregated state in its memory.
 *
 * @param checkpointPath The path to write checkpoint metadata to. If null, the value from
 *                       the runtime configuration will be used.
 * @param savepointPath  The path to write savepoints to. If null, the value from
 *                       the runtime configuration will be used.
 * @param maxStateSize   The maximal size of the serialized state.
 * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous
 *                              snapshot mode. If null, the value configured in the
 *                              runtime configuration will be used.
 */
public MemoryStateBackend(
		@Nullable String checkpointPath,
		@Nullable String savepointPath,
		int maxStateSize,
		TernaryBoolean asynchronousSnapshots) {

	super(checkpointPath == null ? null : new Path(checkpointPath),
			savepointPath == null ? null : new Path(savepointPath));

	checkArgument(maxStateSize > 0, "maxStateSize must be > 0");
	this.maxStateSize = maxStateSize;

	this.asynchronousSnapshots = asynchronousSnapshots;
}
 
Example #29
Source File: FsStateBackend.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new state backend that stores its checkpoint data in the file system and location
 * defined by the given URI.
 *
 * <p>A file system for the file system scheme in the URI (e.g., 'file://', 'hdfs://', or 'S3://')
 * must be accessible via {@link FileSystem#get(URI)}.
 *
 * <p>For a state backend targeting HDFS, this means that the URI must either specify the authority
 * (host and port), or that the Hadoop configuration that describes that information must be in the
 * classpath.
 *
 * @param checkpointDataUri The URI describing the filesystem (scheme and optionally authority),
 *                          and the path to the checkpoint data directory.
 * @param fileStateSizeThreshold State up to this size will be stored as part of the metadata,
 *                             rather than in files (-1 for default value).
 * @param asynchronousSnapshots Switch to enable asynchronous snapshots.
 */
public FsStateBackend(
		URI checkpointDataUri,
		int fileStateSizeThreshold,
		boolean asynchronousSnapshots) {

	this(checkpointDataUri, null, fileStateSizeThreshold,
			TernaryBoolean.fromBoolean(asynchronousSnapshots));
}
 
Example #30
Source File: MemoryStateBackend.java    From Flink-CEPplus with Apache License 2.0 3 votes vote down vote up
/**
 * Creates a new MemoryStateBackend, setting optionally the paths to persist checkpoint metadata
 * and savepoints to, as well as configuring state thresholds and asynchronous operations.
 *
 * <p><b>WARNING:</b> Increasing the size of this value beyond the default value
 * ({@value #DEFAULT_MAX_STATE_SIZE}) should be done with care.
 * The checkpointed state needs to be send to the JobManager via limited size RPC messages, and there
 * and the JobManager needs to be able to hold all aggregated state in its memory.
 *
 * @param checkpointPath The path to write checkpoint metadata to. If null, the value from
 *                       the runtime configuration will be used.
 * @param savepointPath  The path to write savepoints to. If null, the value from
 *                       the runtime configuration will be used.
 * @param maxStateSize   The maximal size of the serialized state.
 * @param asynchronousSnapshots Flag to switch between synchronous and asynchronous
 *                              snapshot mode. If null, the value configured in the
 *                              runtime configuration will be used.
 */
public MemoryStateBackend(
		@Nullable String checkpointPath,
		@Nullable String savepointPath,
		int maxStateSize,
		TernaryBoolean asynchronousSnapshots) {

	super(checkpointPath == null ? null : new Path(checkpointPath),
			savepointPath == null ? null : new Path(savepointPath));

	checkArgument(maxStateSize > 0, "maxStateSize must be > 0");
	this.maxStateSize = maxStateSize;

	this.asynchronousSnapshots = asynchronousSnapshots;
}