Java Code Examples for org.apache.flink.configuration.Configuration#setLong()

@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(ConfigConstants.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 

/**
 * Tests that interrupt happens via watch dog if canceller is stuck in cancel.
 * Task cancellation blocks the task canceller. Interrupt after cancel via
 * cancellation watch dog.
 */
@Test
public void testWatchDogInterruptsTask() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL.key(), 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT.key(), 60 * 1000);

	final Task task = createTaskBuilder()
		.setInvokable(InvokableBlockingInCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 

@Test
public void testConfigurationParsing() {
	final Configuration configuration = new Configuration();

	final long initialRegistrationTimeout = 1L;
	final long maxRegistrationTimeout = 2L;
	final long refusedRegistrationDelay = 3L;
	final long errorRegistrationDelay = 4L;

	configuration.setLong(ClusterOptions.INITIAL_REGISTRATION_TIMEOUT, initialRegistrationTimeout);
	configuration.setLong(ClusterOptions.MAX_REGISTRATION_TIMEOUT, maxRegistrationTimeout);
	configuration.setLong(ClusterOptions.REFUSED_REGISTRATION_DELAY, refusedRegistrationDelay);
	configuration.setLong(ClusterOptions.ERROR_REGISTRATION_DELAY, errorRegistrationDelay);

	final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

	assertThat(retryingRegistrationConfiguration.getInitialRegistrationTimeoutMillis(), is(initialRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getMaxRegistrationTimeoutMillis(), is(maxRegistrationTimeout));
	assertThat(retryingRegistrationConfiguration.getRefusedDelayMillis(), is(refusedRegistrationDelay));
	assertThat(retryingRegistrationConfiguration.getErrorDelayMillis(), is(errorRegistrationDelay));
}
 

@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	S3TestCredentials.assumeCredentialsAvailable();

	basePath = new Path(S3TestCredentials.getTestBucketUri() + "tests-" + UUID.randomUUID());

	// initialize configuration with valid credentials
	final Configuration conf = new Configuration();
	conf.setString("s3.access.key", S3TestCredentials.getS3AccessKey());
	conf.setString("s3.secret.key", S3TestCredentials.getS3SecretKey());

	conf.setLong(PART_UPLOAD_MIN_SIZE, PART_UPLOAD_MIN_SIZE_VALUE);
	conf.setInteger(MAX_CONCURRENT_UPLOADS, MAX_CONCURRENT_UPLOADS_VALUE);

	final String defaultTmpDir = TEMP_FOLDER.getRoot().getAbsolutePath() + "s3_tmp_dir";
	conf.setString(CoreOptions.TMP_DIRS, defaultTmpDir);

	FileSystem.initialize(conf);

	skipped = false;
}
 

/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = new TaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 

@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 

/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = createTaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 

@Test
public void testBundleTime() {
	final Configuration configuration = new Configuration();
	final long defaultBundleTime = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS);
	assertThat(defaultBundleTime, is(equalTo(PythonOptions.MAX_BUNDLE_TIME_MILLS.defaultValue())));

	final long expectedBundleTime = 100;
	configuration.setLong(PythonOptions.MAX_BUNDLE_TIME_MILLS, expectedBundleTime);

	final long actualBundleSize = configuration.getLong(PythonOptions.MAX_BUNDLE_TIME_MILLS);
	assertThat(actualBundleSize, is(equalTo(expectedBundleTime)));
}
 

private static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL);

	return config;
}
 

public PythonPlanBinder(Configuration globalConfig) {
	String configuredPlanTmpPath = globalConfig.getString(PythonOptions.PLAN_TMP_DIR);
	tmpPlanFilesDir = configuredPlanTmpPath != null
		? configuredPlanTmpPath
		: System.getProperty("java.io.tmpdir") + File.separator + "flink_plan_" + UUID.randomUUID();

	operatorConfig = new Configuration();
	operatorConfig.setString(PythonOptions.PYTHON_BINARY_PATH, globalConfig.getString(PythonOptions.PYTHON_BINARY_PATH));
	String configuredTmpDataDir = globalConfig.getString(PythonOptions.DATA_TMP_DIR);
	if (configuredTmpDataDir != null) {
		operatorConfig.setString(PythonOptions.DATA_TMP_DIR, configuredTmpDataDir);
	}
	operatorConfig.setLong(PythonOptions.MMAP_FILE_SIZE, globalConfig.getLong(PythonOptions.MMAP_FILE_SIZE));
}
 

private static Configuration getConfiguration() {
	Configuration config = new Configuration();
	config.setString(AkkaOptions.ASK_TIMEOUT, TestingUtils.DEFAULT_AKKA_ASK_TIMEOUT());
	config.setLong(HeartbeatManagerOptions.HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL);

	return config;
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

private Configuration createTestConfiguration(boolean cleanupExpiredJobs) {
	Configuration historyServerConfig = new Configuration();
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_DIRS, jmDirectory.toURI().toString());
	historyServerConfig.setString(HistoryServerOptions.HISTORY_SERVER_WEB_DIR, hsDirectory.getAbsolutePath());
	historyServerConfig.setLong(HistoryServerOptions.HISTORY_SERVER_ARCHIVE_REFRESH_INTERVAL, 100L);

	historyServerConfig.setBoolean(HistoryServerOptions.HISTORY_SERVER_CLEANUP_EXPIRED_JOBS, cleanupExpiredJobs);

	historyServerConfig.setInteger(HistoryServerOptions.HISTORY_SERVER_WEB_PORT, 0);
	return historyServerConfig;
}
 

/**
 * Tests that {@link SlotManagerConfiguration#getSlotRequestTimeout()} returns the value
 * configured under key {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT}.
 */
@Test
public void testSetSlotRequestTimeout() throws Exception {
	final long slotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, slotIdleTimeout);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(slotIdleTimeout)));
}
 

/**
 * Tests that {@link ResourceManagerOptions#SLOT_REQUEST_TIMEOUT} is preferred over
 * {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT} if set.
 */
@Test
public void testPreferLegacySlotRequestTimeout() throws Exception {
	final long legacySlotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT, legacySlotIdleTimeout);
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 300000L);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(legacySlotIdleTimeout)));
}
 

@Before
public void setUp() throws Exception {
	final Configuration clientConfig = new Configuration();
	clientConfig.setInteger(RestOptions.RETRY_MAX_ATTEMPTS, 0);
	clientConfig.setLong(RestOptions.RETRY_DELAY, 0);
	clientConfig.addAll(CLUSTER.getClientConfiguration());

	client = new RestClusterClient<>(
		clientConfig,
		StandaloneClusterId.getInstance()
	);
}
 

@Test
public void testGetExternalResourcesWithConfigKeyNotSpecifiedOrEmpty() {
	final Configuration config = new Configuration();
	final String resourceConfigKey = "";

	config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, RESOURCE_LIST);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_1), RESOURCE_AMOUNT_1);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_2), RESOURCE_AMOUNT_2);
	config.setString(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(RESOURCE_NAME_1, SUFFIX), resourceConfigKey);

	final Map<String, Long> configMap = ExternalResourceUtils.getExternalResources(config, SUFFIX);

	assertThat(configMap.entrySet(), is(empty()));
}
 

@Test
public void testGetExternalResourcesWithIllegalAmount() {
	final Configuration config = new Configuration();
	final long resourceAmount = 0L;

	config.set(ExternalResourceOptions.EXTERNAL_RESOURCE_LIST, RESOURCE_LIST);
	config.setLong(ExternalResourceOptions.getAmountConfigOptionForResource(RESOURCE_NAME_1), resourceAmount);
	config.setString(ExternalResourceOptions.getSystemConfigKeyConfigOptionForResource(RESOURCE_NAME_1, SUFFIX), RESOURCE_CONFIG_KEY_1);

	final Map<String, Long> configMap = ExternalResourceUtils.getExternalResources(config, SUFFIX);

	assertThat(configMap.entrySet(), is(empty()));
}
 

/**
 * Tests that {@link TransientBlobCache} cleans up after a default TTL and keeps files which are
 * constantly accessed.
 */
private void testTransientBlobCleanup(@Nullable final JobID jobId)
		throws IOException, InterruptedException, ExecutionException {

	// 1s should be a safe-enough buffer to still check for existence after a BLOB's last access
	long cleanupInterval = 1L; // in seconds
	final int numberConcurrentGetOperations = 3;

	final List<CompletableFuture<Void>> getOperations = new ArrayList<>(numberConcurrentGetOperations);

	byte[] data = new byte[2000000];
	rnd.nextBytes(data);
	byte[] data2 = Arrays.copyOfRange(data, 10, 54);

	Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setLong(BlobServerOptions.CLEANUP_INTERVAL, cleanupInterval);

	long cleanupLowerBound;

	try (
		BlobServer server = new BlobServer(config, new VoidBlobStore());
		final BlobCacheService cache = new BlobCacheService(
			config, new VoidBlobStore(), new InetSocketAddress("localhost", server.getPort())
		)) {
		ConcurrentMap<Tuple2<JobID, TransientBlobKey>, Long> transientBlobExpiryTimes =
			cache.getTransientBlobService().getBlobExpiryTimes();

		server.start();

		final TransientBlobKey key1 =
			(TransientBlobKey) put(server, jobId, data, TRANSIENT_BLOB);
		final TransientBlobKey key2 =
			(TransientBlobKey) put(server, jobId, data2, TRANSIENT_BLOB);

		// access key1, verify expiry times
		cleanupLowerBound = System.currentTimeMillis() + cleanupInterval;
		verifyContents(cache, jobId, key1, data);
		final Long key1ExpiryFirstAccess = transientBlobExpiryTimes.get(Tuple2.of(jobId, key1));
		assertThat(key1ExpiryFirstAccess, greaterThanOrEqualTo(cleanupLowerBound));
		assertNull(transientBlobExpiryTimes.get(Tuple2.of(jobId, key2)));

		// access key2, verify expiry times (delay at least 1ms to also verify key1 expiry is unchanged)
		Thread.sleep(1);
		cleanupLowerBound = System.currentTimeMillis() + cleanupInterval;
		verifyContents(cache, jobId, key2, data2);
		assertEquals(key1ExpiryFirstAccess, transientBlobExpiryTimes.get(Tuple2.of(jobId, key1)));
		assertThat(transientBlobExpiryTimes.get(Tuple2.of(jobId, key2)),
			greaterThanOrEqualTo(cleanupLowerBound));

		// files are cached now for the given TTL - remove from server so that they are not re-downloaded
		if (jobId != null) {
			server.cleanupJob(jobId, true);
		} else {
			server.deleteFromCache(key1);
			server.deleteFromCache(key2);
		}
		checkFileCountForJob(0, jobId, server);

		// cleanup task is run every cleanupInterval seconds
		// => unaccessed file should remain at most 2*cleanupInterval seconds
		// (use 3*cleanupInterval to check that we can still access it)
		final long finishTime = System.currentTimeMillis() + 3 * cleanupInterval;

		final ExecutorService executor = Executors.newFixedThreadPool(numberConcurrentGetOperations);
		for (int i = 0; i < numberConcurrentGetOperations; i++) {
			CompletableFuture<Void> getOperation = CompletableFuture
				.supplyAsync(
					() -> {
						try {
							// constantly access key1 so this should not get deleted
							while (System.currentTimeMillis() < finishTime) {
								get(cache, jobId, key1);
							}

							return null;
						} catch (IOException e) {
							throw new CompletionException(new FlinkException(
								"Could not retrieve blob.", e));
						}
				}, executor);

			getOperations.add(getOperation);
		}

		FutureUtils.ConjunctFuture<Collection<Void>> filesFuture = FutureUtils.combineAll(getOperations);
		filesFuture.get();

		verifyDeletedEventually(server, jobId, key1, key2);
	}
}
 

private Configuration getConfig() {
	Configuration config = new Configuration();
	config.setInteger(SourceReaderOptions.ELEMENT_QUEUE_CAPACITY, 1);
	config.setLong(SourceReaderOptions.SOURCE_READER_CLOSE_TIMEOUT, 30000L);
	return config;
}