Java Code Examples for org.apache.flink.configuration.Configuration#setLong()

The following examples show how to use org.apache.flink.configuration.Configuration#setLong() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   File: BlobsCleanupITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(ConfigConstants.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 
Example 2
Source Project: flink   File: TaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = createTaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 
Example 3
Source Project: flink   File: BlobsCleanupITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 
Example 4
Source Project: Flink-CEPplus   File: TaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = new TaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 
Example 5
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	S3TestCredentials.assumeCredentialsAvailable();

	basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID());

	// initialize configuration with valid credentials
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
	conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

	conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE);
	conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE);

	final String defaultTmpDir = TEMP_FOLDER.getRoot().getAbsolutePath() + "s3_tmp_dir";
	conf.setString(CoreOptions.TMP_DIRS, defaultTmpDir);

	FileSystem.initialize(conf);

	skipped = false;
}
 
Example 6
@Test
public void testConfigurationParsing() {
	final Configuration configuration = new Configuration();

	final long initialRegistrationTimeout = 1L;
	final long maxRegistrationTimeout = 2L;
	final long refusedRegistrationDelay = 3L;
	final long errorRegistrationDelay = 4L;

	configuration.setLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT, initialRegistrationTimeout);
	configuration.setLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT, maxRegistrationTimeout);
	configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, refusedRegistrationDelay);
	configuration.setLong(ClusterOptions.ERROR_REGISTRATION_DELAY, errorRegistrationDelay);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(initialRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(maxRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(refusedRegistrationDelay));
	assertThat(retryingRegistrationConfiguration.getErrorDelayMillis(), is(errorRegistrationDelay));
}
 
Example 7
Source Project: flink   File: TaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that interrupt happens via watch dog if canceller is stuck in cancel.
 * Task cancellation blocks the task canceller. Interrupt after cancel via
 * cancellation watch dog.
 */
@Test
public void testWatchDogInterruptsTask() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL.key(), 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT.key(), 60 * 1000);

	final Task task = createTaskBuilder()
		.setInvokable(InvokableBlockingInCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 
Example 8
Source Project: flink   File: ExternalResourceUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetExternalResourcesWithIllegalAmount() {
	final Configuration config = new Configuration();
	final long resourceAmount = 0L;

	config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, RESOURCE_LIST);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_1), resourceAmount);
	config.setString(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(RESOURCE_NAME_1, SUFFIX), RESOURCE_CONFIG_KEY_1);

	final Map<String, Long> configMap = ExternalResourceUtils.getExternalResources(config, SUFFIX);

	assertThat(configMap.entrySet(), is(empty()));
}
 
Example 9
Source Project: flink   File: ExternalResourceUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetExternalResourcesWithConfigKeyNotSpecifiedOrEmpty() {
	final Configuration config = new Configuration();
	final String resourceConfigKey = "";

	config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, RESOURCE_LIST);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_1), RESOURCE_AMOUNT_1);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_2), RESOURCE_AMOUNT_2);
	config.setString(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(RESOURCE_NAME_1, SUFFIX), resourceConfigKey);

	final Map<String, Long> configMap = ExternalResourceUtils.getExternalResources(config, SUFFIX);

	assertThat(configMap.entrySet(), is(empty()));
}
 
Example 10
Source Project: flink   File: HistoryServerTest.java    License: Apache License 2.0 5 votes vote down vote up
private Configuration createTestConfiguration(boolean cleanupExpiredJobs) {
	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());
	historyServerConfig.setLong(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_REFRESH_INTERVAL, 100L);

	historyServerConfig.setBoolean(HistoryServerOptions.HISTORY_SERVER_CLEANUP_EXPIRED_JOBS, cleanupExpiredJobs);

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);
	return historyServerConfig;
}
 
Example 11
Source Project: flink   File: AccumulatorLiveITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL);

	return config;
}
 
Example 12
Source Project: Flink-CEPplus   File: PythonPlanBinder.java    License: Apache License 2.0 5 votes vote down vote up
public PythonPlanBinder(Configuration globalConfig) {
	String configuredPlanTmpPath = globalConfig.getString(PythonOptions.PLAN_TMP_DIR);
	tmpPlanFilesDir = configuredPlanTmpPath != null
		? configuredPlanTmpPath
		: System.getProperty("java.io.tmpdir") + File.separator + "flink_plan_" + UUID.randomUUID();

	operatorConfig = new Configuration();
	operatorConfig.setString(PythonOptions.PYTHON_BINARY_PATH, globalConfig.getString(PythonOptions.PYTHON_BINARY_PATH));
	String configuredTmpDataDir = globalConfig.getString(PythonOptions.DATA_TMP_DIR);
	if (configuredTmpDataDir != null) {
		operatorConfig.setString(PythonOptions.DATA_TMP_DIR, configuredTmpDataDir);
	}
	operatorConfig.setLong(PythonOptions.MMAP_FILE_SIZE, globalConfig.getLong(PythonOptions.MMAP_FILE_SIZE));
}
 
Example 13
Source Project: flink   File: AccumulatorLiveITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL);

	return config;
}
 
Example 14
Source Project: flink   File: PythonOptionsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBundleTime() {
	final Configuration configuration = new Configuration();
	final long defaultBundleTime = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS);
	assertThat(defaultBundleTime, is(equalTo(PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue())));

	final long expectedBundleTime = 100;
	configuration.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, expectedBundleTime);

	final long actualBundleSize = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS);
	assertThat(actualBundleSize, is(equalTo(expectedBundleTime)));
}
 
Example 15
Source Project: flink   File: JobRetrievalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 
Example 16
/**
 * Tests that {@link SlotManagerConfiguration#getSlotRequestTimeout()} returns the value
 * configured under key {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT}.
 */
@Test
public void testSetSlotRequestTimeout() throws Exception {
	final long slotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, slotIdleTimeout);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(slotIdleTimeout)));
}
 
Example 17
/**
 * Tests that {@link ResourceManagerOptions#SLOT_REQUEST_TIMEOUT} is preferred over
 * {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT} if set.
 */
@Test
public void testPreferLegacySlotRequestTimeout() throws Exception {
	final long legacySlotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT, legacySlotIdleTimeout);
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 300000L);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(legacySlotIdleTimeout)));
}
 
Example 18
Source Project: flink   File: JobRetrievalITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 
Example 19
Source Project: flink   File: BlobCacheCleanupTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that {@link TransientBlobCache} cleans up after a default TTL and keeps files which are
 * constantly accessed.
 */
private void testTransientBlobCleanup(@Nullable final JobID jobId)
		throws IOException, InterruptedException, ExecutionException {

	// 1s should be a safe-enough buffer to still check for existence after a BLOB's last access
	long cleanupInterval = 1L; // in seconds
	final int numberConcurrentGetOperations = 3;

	final List<CompletableFuture<Void>> getOperations = new ArrayList<>(numberConcurrentGetOperations);

	byte[] data = new byte[2000000];
	rnd.nextBytes(data);
	byte[] data2 = Arrays.copyOfRange(data, 10, 54);

	Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setLong(BlobServerOptions.CLEANUP_INTERVAL, cleanupInterval);

	long cleanupLowerBound;

	try (
		BlobServer server = new BlobServer(config, new VoidBlobStore());
		final BlobCacheService cache = new BlobCacheService(
			config, new VoidBlobStore(), new InetSocketAddress("localhost", server.getPort())
		)) {
		ConcurrentMap<Tuple2<JobID, TransientBlobKey>, Long> transientBlobExpiryTimes =
			cache.getTransientBlobService().getBlobExpiryTimes();

		server.start();

		final TransientBlobKey key1 =
			(TransientBlobKey) put(server, jobId, data, TRANSIENT_BLOB);
		final TransientBlobKey key2 =
			(TransientBlobKey) put(server, jobId, data2, TRANSIENT_BLOB);

		// access key1, verify expiry times
		cleanupLowerBound = System.currentTimeMillis() + cleanupInterval;
		verifyContents(cache, jobId, key1, data);
		final Long key1ExpiryFirstAccess = transientBlobExpiryTimes.get(Tuple2.of(jobId, key1));
		assertThat(key1ExpiryFirstAccess, greaterThanOrEqualTo(cleanupLowerBound));
		assertNull(transientBlobExpiryTimes.get(Tuple2.of(jobId, key2)));

		// access key2, verify expiry times (delay at least 1ms to also verify key1 expiry is unchanged)
		Thread.sleep(1);
		cleanupLowerBound = System.currentTimeMillis() + cleanupInterval;
		verifyContents(cache, jobId, key2, data2);
		assertEquals(key1ExpiryFirstAccess, transientBlobExpiryTimes.get(Tuple2.of(jobId, key1)));
		assertThat(transientBlobExpiryTimes.get(Tuple2.of(jobId, key2)),
			greaterThanOrEqualTo(cleanupLowerBound));

		// files are cached now for the given TTL - remove from server so that they are not re-downloaded
		if (jobId != null) {
			server.cleanupJob(jobId, true);
		} else {
			server.deleteFromCache(key1);
			server.deleteFromCache(key2);
		}
		checkFileCountForJob(0, jobId, server);

		// cleanup task is run every cleanupInterval seconds
		// => unaccessed file should remain at most 2*cleanupInterval seconds
		// (use 3*cleanupInterval to check that we can still access it)
		final long finishTime = System.currentTimeMillis() + 3 * cleanupInterval;

		final ExecutorService executor = Executors.newFixedThreadPool(numberConcurrentGetOperations);
		for (int i = 0; i < numberConcurrentGetOperations; i++) {
			CompletableFuture<Void> getOperation = CompletableFuture
				.supplyAsync(
					() -> {
						try {
							// constantly access key1 so this should not get deleted
							while (System.currentTimeMillis() < finishTime) {
								get(cache, jobId, key1);
							}

							return null;
						} catch (IOException e) {
							throw new CompletionException(new FlinkException(
								"Could not retrieve blob.", e));
						}
				}, executor);

			getOperations.add(getOperation);
		}

		FutureUtils.ConjunctFuture<Collection<Void>> filesFuture = FutureUtils.combineAll(getOperations);
		filesFuture.get();

		verifyDeletedEventually(server, jobId, key1, key2);
	}
}
 
Example 20
Source Project: flink   File: SourceReaderBaseTest.java    License: Apache License 2.0 4 votes vote down vote up
private Configuration getConfig() {
	Configuration config = new Configuration();
	config.setInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY, 1);
	config.setLong(SourceReaderOptions.SOURCE_READER_CLOSE_TIMEOUT, 30000L);
	return config;
}