org.apache.flink.configuration.ResourceManagerOptions Java Examples

/**
 * See documentation.
 */
public static int calculateHeapSize(int memory, org.apache.flink.configuration.Configuration conf) {

	float memoryCutoffRatio = conf.getFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);
	int minCutoff = conf.getInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (memoryCutoffRatio > 1 || memoryCutoffRatio < 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key()
			+ "' must be between 0 and 1. Value given=" + memoryCutoffRatio);
	}
	if (minCutoff > memory) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key()
			+ "' is higher (" + minCutoff + ") than the requested amount of memory " + memory);
	}

	int heapLimit = (int) ((float) memory * memoryCutoffRatio);
	if (heapLimit < minCutoff) {
		heapLimit = minCutoff;
	}
	return memory - heapLimit;
}
 

public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(
		Configuration configuration,
		WorkerResourceSpecFactory defaultWorkerResourceSpecFactory) throws ConfigurationException {

	final String strJobTimeout = configuration.getString(ResourceManagerOptions.JOB_TIMEOUT);
	final Time jobTimeout;

	try {
		jobTimeout = Time.milliseconds(TimeUtils.parseDuration(strJobTimeout).toMillis());
	} catch (IllegalArgumentException e) {
		throw new ConfigurationException("Could not parse the resource manager's job timeout " +
			"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
	}

	final WorkerResourceSpec defaultWorkerResourceSpec = defaultWorkerResourceSpecFactory.createDefaultWorkerResourceSpec(configuration);
	final SlotManagerConfiguration slotManagerConfiguration =
		SlotManagerConfiguration.fromConfiguration(configuration, defaultWorkerResourceSpec);

	return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
}
 

/**
 * Computes the parameters to be used to start a TaskManager Java process.
 *
 * @param config The Flink configuration.
 * @param taskExecutorProcessSpec The resource specifics of the task executor.
 * @return The parameters to start the TaskManager processes with.
 */
public static ContaineredTaskManagerParameters create(
		Configuration config,
		TaskExecutorProcessSpec taskExecutorProcessSpec) {

	// obtain the additional environment variables from the configuration
	final HashMap<String, String> envVars = new HashMap<>();
	final String prefix = ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX;

	for (String key : config.keySet()) {
		if (key.startsWith(prefix) && key.length() > prefix.length()) {
			// remove prefix
			String envVarKey = key.substring(prefix.length());
			envVars.put(envVarKey, config.getString(key, null));
		}
	}

	// done
	return new ContaineredTaskManagerParameters(taskExecutorProcessSpec, envVars);
}
 

@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);

	flinkConfiguration.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	final int minMemory = 100;
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, minMemory);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 

@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 

/**
 * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}.
 */
@Test
public void testCalculateCutoffMB() {

	Configuration config = new Configuration();
	long containerMemoryMB = 1000L;

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f);
	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128);

	assertEquals(128,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.2f);
	assertEquals(200,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 1000);

	try {
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB);
		fail("Expected to fail with an invalid argument exception.");
	} catch (IllegalArgumentException ignored) {
		// we expected it.
	}
}
 

public static SlotManagerConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
	final String strTimeout = configuration.getString(AkkaOptions.ASK_TIMEOUT);
	final Time rpcTimeout;

	try {
		rpcTimeout = Time.milliseconds(Duration.apply(strTimeout).toMillis());
	} catch (NumberFormatException e) {
		throw new ConfigurationException("Could not parse the resource manager's timeout " +
			"value " + AkkaOptions.ASK_TIMEOUT + '.', e);
	}

	final Time slotRequestTimeout = getSlotRequestTimeout(configuration);
	final Time taskManagerTimeout = Time.milliseconds(
			configuration.getLong(ResourceManagerOptions.TASK_MANAGER_TIMEOUT));

	boolean waitResultConsumedBeforeRelease =
		configuration.getBoolean(ResourceManagerOptions.TASK_MANAGER_RELEASE_WHEN_RESULT_CONSUMED);

	return new SlotManagerConfiguration(rpcTimeout, slotRequestTimeout, taskManagerTimeout, waitResultConsumedBeforeRelease);
}
 

public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {

		final String strJobTimeout = configuration.getString(ResourceManagerOptions.JOB_TIMEOUT);
		final Time jobTimeout;

		try {
			jobTimeout = Time.milliseconds(Duration.apply(strJobTimeout).toMillis());
		} catch (NumberFormatException e) {
			throw new ConfigurationException("Could not parse the resource manager's job timeout " +
				"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
		}

		final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

		return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
	}
 

@Before
public void setup() {
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	flinkConfig = new Configuration();
	flinkConfig.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 100);

	File root = folder.getRoot();
	File home = new File(root, "home");
	boolean created = home.mkdir();
	assertTrue(created);

	env = new HashMap<>();
	env.put(ENV_APP_ID, "foo");
	env.put(ENV_CLIENT_HOME_DIR, home.getAbsolutePath());
	env.put(ENV_CLIENT_SHIP_FILES, "");
	env.put(ENV_FLINK_CLASSPATH, "");
	env.put(ENV_HADOOP_USER_NAME, "foo");
	env.put(FLINK_JAR_PATH, root.toURI().toString());
}
 

/**
 * See documentation.
 */
public static int calculateHeapSize(int memory, org.apache.flink.configuration.Configuration conf) {

	float memoryCutoffRatio = conf.getFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);
	int minCutoff = conf.getInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (memoryCutoffRatio > 1 || memoryCutoffRatio < 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key()
			+ "' must be between 0 and 1. Value given=" + memoryCutoffRatio);
	}
	if (minCutoff > memory) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key()
			+ "' is higher (" + minCutoff + ") than the requested amount of memory " + memory);
	}

	int heapLimit = (int) ((float) memory * memoryCutoffRatio);
	if (heapLimit < minCutoff) {
		heapLimit = minCutoff;
	}
	return memory - heapLimit;
}
 

@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);

	flinkConfiguration.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	final int minMemory = 100;
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, minMemory);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 

/**
 * Test to guard {@link ContaineredTaskManagerParameters#calculateCutoffMB(Configuration, long)}.
 */
@Test
public void testCalculateCutoffMB() {

	Configuration config = new Configuration();
	long containerMemoryMB = 1000L;

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.1f);
	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 128);

	assertEquals(128,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setFloat(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO, 0.2f);
	assertEquals(200,
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB));

	config.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 1000);

	try {
		ContaineredTaskManagerParameters.calculateCutoffMB(config, containerMemoryMB);
		fail("Expected to fail with an invalid argument exception.");
	} catch (IllegalArgumentException ignored) {
		// we expected it.
	}
}
 

public static ResourceManagerRuntimeServicesConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {

		final String strJobTimeout = configuration.getString(ResourceManagerOptions.JOB_TIMEOUT);
		final Time jobTimeout;

		try {
			jobTimeout = Time.milliseconds(Duration.apply(strJobTimeout).toMillis());
		} catch (NumberFormatException e) {
			throw new ConfigurationException("Could not parse the resource manager's job timeout " +
				"value " + ResourceManagerOptions.JOB_TIMEOUT + '.', e);
		}

		final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

		return new ResourceManagerRuntimeServicesConfiguration(jobTimeout, slotManagerConfiguration);
	}
 

@Before
public void setup() {
	testingFatalErrorHandler = new TestingFatalErrorHandler();

	flinkConfig = new Configuration();
	flinkConfig.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 100);

	File root = folder.getRoot();
	File home = new File(root, "home");
	boolean created = home.mkdir();
	assertTrue(created);

	env = new HashMap<>();
	env.put(ENV_APP_ID, "foo");
	env.put(ENV_CLIENT_HOME_DIR, home.getAbsolutePath());
	env.put(ENV_CLIENT_SHIP_FILES, "");
	env.put(ENV_FLINK_CLASSPATH, "");
	env.put(ENV_HADOOP_USER_NAME, "foo");
	env.put(FLINK_JAR_PATH, root.toURI().toString());
}
 

public static SlotManagerConfiguration fromConfiguration(Configuration configuration) throws ConfigurationException {
	final String strTimeout = configuration.getString(AkkaOptions.ASK_TIMEOUT);
	final Time rpcTimeout;

	try {
		rpcTimeout = Time.milliseconds(Duration.apply(strTimeout).toMillis());
	} catch (NumberFormatException e) {
		throw new ConfigurationException("Could not parse the resource manager's timeout " +
			"value " + AkkaOptions.ASK_TIMEOUT + '.', e);
	}

	final Time slotRequestTimeout = getSlotRequestTimeout(configuration);
	final Time taskManagerTimeout = Time.milliseconds(
			configuration.getLong(ResourceManagerOptions.TASK_MANAGER_TIMEOUT));

	boolean waitResultConsumedBeforeRelease =
		configuration.getBoolean(ResourceManagerOptions.TASK_MANAGER_RELEASE_WHEN_RESULT_CONSUMED);

	return new SlotManagerConfiguration(rpcTimeout, slotRequestTimeout, taskManagerTimeout, waitResultConsumedBeforeRelease);
}
 

/**
 * Tests that {@link ResourceManagerOptions#SLOT_REQUEST_TIMEOUT} is preferred over
 * {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT} if set.
 */
@Test
public void testPreferLegacySlotRequestTimeout() throws Exception {
	final long legacySlotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT, legacySlotIdleTimeout);
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 300000L);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(legacySlotIdleTimeout)));
}
 

/**
 * Tests that {@link ResourceManagerOptions#SLOT_REQUEST_TIMEOUT} is preferred over
 * {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT} if set.
 */
@Test
public void testPreferLegacySlotRequestTimeout() throws Exception {
	final long legacySlotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT, legacySlotIdleTimeout);
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 300000L);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(legacySlotIdleTimeout)));
}
 

@Test
public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = new YarnClusterDescriptor(
		flinkConfiguration,
		yarnConfiguration,
		temporaryFolder.getRoot().getAbsolutePath(),
		yarnClient,
		true);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(Integer.MAX_VALUE)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}
 

/**
 * Test per-job yarn cluster and memory calculations for off-heap use (see FLINK-7400) with the
 * same job as {@link #perJobYarnCluster()}.
 *
 * <p>This ensures that with (any) pre-allocated off-heap memory by us, there is some off-heap
 * memory remaining for Flink's libraries. Creating task managers will thus fail if no off-heap
 * memory remains.
 */
@Test
public void perJobYarnClusterOffHeap() throws Exception {
	runTest(() -> {
		LOG.info("Starting perJobYarnCluster()");
		addTestAppender(CliFrontend.class, Level.INFO);
		File exampleJarLocation = getTestJarPath("BatchWordCount.jar");

		// set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
		final long taskManagerMemoryMB = 1024;
		//noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
		final long networkBuffersMB = NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(
			(taskManagerMemoryMB - ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
			new Configuration()) >> 20;
		final long offHeapMemory = taskManagerMemoryMB
			- ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
			// cutoff memory (will be added automatically)
			- networkBuffersMB // amount of memory used for network buffers
			- 100; // reserve something for the Java heap space

		runWithArgs(new String[]{"run", "-m", "yarn-cluster",
				"-yj", flinkUberjar.getAbsolutePath(),
				"-yt", flinkLibFolder.getAbsolutePath(),
				"-yt", flinkShadedHadoopDir.getAbsolutePath(),
				"-yn", "1",
				"-ys", "2", //test that the job is executed with a DOP of 2
				"-yjm", "768m",
				"-ytm", taskManagerMemoryMB + "m",
				"-yD", "taskmanager.memory.off-heap=true",
				"-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
				"-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
			/* test succeeded after this string */
			"Program execution finished",
			/* prohibited strings: (to verify the parallelism) */
			// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
			new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
			RunTypes.CLI_FRONTEND, 0, true);
		LOG.info("Finished perJobYarnCluster()");
	});
}
 

@Override
protected void setupFlinkConfig() {
	super.setupFlinkConfig();

	flinkConfig.set(TaskManagerOptions.CPU_CORES, TASK_MANAGER_CPU);
	flinkConfig.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.parse(TASK_MANAGER_MEMORY + "m"));
	flinkConfig.set(TaskManagerOptions.RPC_PORT, String.valueOf(RPC_PORT));

	customizedEnvs.forEach((k, v) ->
		flinkConfig.setString(ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX + k, v));
}
 

@Test
public void testGetEnvironments() {
	final Map<String, String> expectedEnvironments = new HashMap<>();
	expectedEnvironments.put("k1", "v1");
	expectedEnvironments.put("k2", "v2");

	expectedEnvironments.forEach((k, v) ->
		flinkConfig.setString(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX + k, v));

	final Map<String, String> resultEnvironments = kubernetesJobManagerParameters.getEnvironments();

	assertEquals(expectedEnvironments, resultEnvironments);
}
 

@Override
protected void setupFlinkConfig() {
	super.setupFlinkConfig();

	this.flinkConfig.set(RestOptions.PORT, REST_PORT);
	this.flinkConfig.set(RestOptions.BIND_PORT, REST_BIND_PORT);
	this.flinkConfig.set(JobManagerOptions.PORT, RPC_PORT);
	this.flinkConfig.set(BlobServerOptions.PORT, Integer.toString(BLOB_SERVER_PORT));
	this.flinkConfig.set(KubernetesConfigOptions.JOB_MANAGER_CPU, JOB_MANAGER_CPU);
	this.customizedEnvs.forEach((k, v) ->
			this.flinkConfig.setString(ResourceManagerOptions.CONTAINERIZED_MASTER_ENV_PREFIX + k, v));
	this.flinkConfig.set(KubernetesConfigOptions.JOB_MANAGER_LABELS, userLabels);
	this.flinkConfig.set(KubernetesConfigOptions.JOB_MANAGER_NODE_SELECTOR, nodeSelector);
	this.flinkConfig.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(JOB_MANAGER_MEMORY));
}
 

@Override
protected void setupFlinkConfig() {
	super.setupFlinkConfig();

	flinkConfig.set(TaskManagerOptions.RPC_PORT, String.valueOf(RPC_PORT));
	flinkConfig.set(TaskManagerOptions.CPU_CORES, TASK_MANAGER_CPU);
	flinkConfig.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.parse(TOTAL_PROCESS_MEMORY + "m"));
	customizedEnvs.forEach((k, v) ->
		flinkConfig.setString(ResourceManagerOptions.CONTAINERIZED_TASK_MANAGER_ENV_PREFIX + k, v));
	this.flinkConfig.set(KubernetesConfigOptions.TASK_MANAGER_LABELS, userLabels);
	this.flinkConfig.set(KubernetesConfigOptions.TASK_MANAGER_NODE_SELECTOR, nodeSelector);
}
 

/**
 * Get the configuration for standalone ResourceManager, overwrite invalid configs.
 *
 * @param configuration configuration object
 * @return the configuration for standalone ResourceManager
 */
private static Configuration getConfigurationWithoutMaxSlotNumberIfSet(Configuration configuration) {
	final Configuration copiedConfig = new Configuration(configuration);
	// The max slot limit should not take effect for standalone cluster, we overwrite the configure in case user
	// sets this value by mistake.
	if (copiedConfig.removeConfig(ResourceManagerOptions.MAX_SLOT_NUM)) {
		LOG.warn("Config option {} will be ignored in standalone mode.", ResourceManagerOptions.MAX_SLOT_NUM.key());
	}
	return copiedConfig;
}
 

public static SlotManagerConfiguration fromConfiguration(
		Configuration configuration,
		WorkerResourceSpec defaultWorkerResourceSpec) throws ConfigurationException {

	final Time rpcTimeout;
	try {
		rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);
	} catch (IllegalArgumentException e) {
		throw new ConfigurationException("Could not parse the resource manager's timeout " +
			"value " + AkkaOptions.ASK_TIMEOUT + '.', e);
	}

	final Time slotRequestTimeout = getSlotRequestTimeout(configuration);
	final Time taskManagerTimeout = Time.milliseconds(
			configuration.getLong(ResourceManagerOptions.TASK_MANAGER_TIMEOUT));

	boolean waitResultConsumedBeforeRelease =
		configuration.getBoolean(ResourceManagerOptions.TASK_MANAGER_RELEASE_WHEN_RESULT_CONSUMED);

	boolean evenlySpreadOutSlots = configuration.getBoolean(ClusterOptions.EVENLY_SPREAD_OUT_SLOTS_STRATEGY);
	final SlotMatchingStrategy slotMatchingStrategy = evenlySpreadOutSlots ?
		LeastUtilizationSlotMatchingStrategy.INSTANCE : AnyMatchingSlotMatchingStrategy.INSTANCE;

	int numSlotsPerWorker = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS);

	int maxSlotNum = configuration.getInteger(ResourceManagerOptions.MAX_SLOT_NUM);

	return new SlotManagerConfiguration(
		rpcTimeout,
		slotRequestTimeout,
		taskManagerTimeout,
		waitResultConsumedBeforeRelease,
		slotMatchingStrategy,
		defaultWorkerResourceSpec,
		numSlotsPerWorker,
		maxSlotNum);
}
 

private static Time getSlotRequestTimeout(final Configuration configuration) {
	final long slotRequestTimeoutMs;
	if (configuration.contains(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT)) {
		LOGGER.warn("Config key {} is deprecated; use {} instead.",
			ResourceManagerOptions.SLOT_REQUEST_TIMEOUT,
			JobManagerOptions.SLOT_REQUEST_TIMEOUT);
		slotRequestTimeoutMs = configuration.getLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT);
	} else {
		slotRequestTimeoutMs = configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT);
	}
	return Time.milliseconds(slotRequestTimeoutMs);
}
 

/**
 * Tests that {@link ResourceManagerOptions#SLOT_REQUEST_TIMEOUT} is preferred over
 * {@link JobManagerOptions#SLOT_REQUEST_TIMEOUT} if set.
 */
@Test
public void testPreferLegacySlotRequestTimeout() throws Exception {
	final long legacySlotIdleTimeout = 42;

	final Configuration configuration = new Configuration();
	configuration.setLong(ResourceManagerOptions.SLOT_REQUEST_TIMEOUT, legacySlotIdleTimeout);
	configuration.setLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT, 300000L);
	final SlotManagerConfiguration slotManagerConfiguration = SlotManagerConfiguration.fromConfiguration(configuration, WorkerResourceSpec.ZERO);

	assertThat(slotManagerConfiguration.getSlotRequestTimeout().toMilliseconds(), is(equalTo(legacySlotIdleTimeout)));
}
 

private SlotManagerBuilder() {
	this.slotMatchingStrategy = AnyMatchingSlotMatchingStrategy.INSTANCE;
	this.scheduledExecutor = TestingUtils.defaultScheduledExecutor();
	this.taskManagerRequestTimeout = TestingUtils.infiniteTime();
	this.slotRequestTimeout = TestingUtils.infiniteTime();
	this.taskManagerTimeout = TestingUtils.infiniteTime();
	this.waitResultConsumedBeforeRelease = true;
	this.defaultWorkerResourceSpec = WorkerResourceSpec.ZERO;
	this.numSlotsPerWorker = 1;
	this.slotManagerMetricGroup = UnregisteredMetricGroups.createUnregisteredSlotManagerMetricGroup();
	this.maxSlotNum = ResourceManagerOptions.MAX_SLOT_NUM.defaultValue();
}
 

/**
 * Calcuate cutoff memory size used by container, it will throw an {@link IllegalArgumentException}
 * if the config is invalid or return the cutoff value if valid.
 *
 * @param config The Flink configuration.
 * @param containerMemoryMB The size of the complete container, in megabytes.
 *
 * @return cutoff memory size used by container.
 */
public static long calculateCutoffMB(Configuration config, long containerMemoryMB) {
	Preconditions.checkArgument(containerMemoryMB > 0);

	// (1) check cutoff ratio
	final float memoryCutoffRatio = config.getFloat(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO);

	if (memoryCutoffRatio >= 1 || memoryCutoffRatio <= 0) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_RATIO.key() + "' must be between 0 and 1. Value given="
			+ memoryCutoffRatio);
	}

	// (2) check min cutoff value
	final int minCutoff = config.getInteger(
		ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN);

	if (minCutoff >= containerMemoryMB) {
		throw new IllegalArgumentException("The configuration value '"
			+ ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.key() + "'='" + minCutoff
			+ "' is larger than the total container memory " + containerMemoryMB);
	}

	// (3) check between heap and off-heap
	long cutoff = (long) (containerMemoryMB * memoryCutoffRatio);
	if (cutoff < minCutoff) {
		cutoff = minCutoff;
	}
	return cutoff;
}
 

/**
 * Test per-job yarn cluster and memory calculations for off-heap use (see FLINK-7400) with the
 * same job as {@link #perJobYarnCluster()}.
 *
 * <p>This ensures that with (any) pre-allocated off-heap memory by us, there is some off-heap
 * memory remaining for Flink's libraries. Creating task managers will thus fail if no off-heap
 * memory remains.
 */
@Test
public void perJobYarnClusterOffHeap() throws IOException {
	LOG.info("Starting perJobYarnCluster()");
	addTestAppender(CliFrontend.class, Level.INFO);
	File exampleJarLocation = getTestJarPath("BatchWordCount.jar");

	// set memory constraints (otherwise this is the same test as perJobYarnCluster() above)
	final long taskManagerMemoryMB = 1024;
	//noinspection NumericOverflow if the calculation of the total Java memory size overflows, default configuration parameters are wrong in the first place, so we can ignore this inspection
	final long networkBuffersMB = TaskManagerServices
		.calculateNetworkBufferMemory(
			(taskManagerMemoryMB -
				ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()) << 20,
			new Configuration()) >> 20;
	final long offHeapMemory = taskManagerMemoryMB
		- ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN.defaultValue()
		// cutoff memory (will be added automatically)
		- networkBuffersMB // amount of memory used for network buffers
		- 100; // reserve something for the Java heap space

	runWithArgs(new String[]{"run", "-m", "yarn-cluster",
			"-yj", flinkUberjar.getAbsolutePath(),
			"-yt", flinkLibFolder.getAbsolutePath(),
			"-yt", flinkShadedHadoopDir.getAbsolutePath(),
			"-yn", "1",
			"-ys", "2", //test that the job is executed with a DOP of 2
			"-yjm", "768m",
			"-ytm", taskManagerMemoryMB + "m",
			"-yD", "taskmanager.memory.off-heap=true",
			"-yD", "taskmanager.memory.size=" + offHeapMemory + "m",
			"-yD", "taskmanager.memory.preallocate=true", exampleJarLocation.getAbsolutePath()},
		/* test succeeded after this string */
		"Program execution finished",
		/* prohibited strings: (to verify the parallelism) */
		// (we should see "DataSink (...) (1/2)" and "DataSink (...) (2/2)" instead)
		new String[]{"DataSink \\(.*\\) \\(1/1\\) switched to FINISHED"},
		RunTypes.CLI_FRONTEND, 0, true);
	LOG.info("Finished perJobYarnCluster()");
}