org.apache.flink.client.deployment.ClusterSpecification Java Examples

The following examples show how to use org.apache.flink.client.deployment.ClusterSpecification. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: YARNApplicationITCase.java    From flink with Apache License 2.0 7 votes vote down vote up
private void deployApplication(Configuration configuration) throws Exception {
	try (final YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor(configuration)) {

		final int masterMemory = yarnClusterDescriptor.getFlinkConfiguration().get(JobManagerOptions.TOTAL_PROCESS_MEMORY).getMebiBytes();
		final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
			.setMasterMemoryMB(masterMemory)
			.setTaskManagerMemoryMB(1024)
			.setSlotsPerTaskManager(1)
			.createClusterSpecification();

		try (ClusterClient<ApplicationId> clusterClient = yarnClusterDescriptor
				.deployApplicationCluster(
						clusterSpecification,
						ApplicationConfiguration.fromConfiguration(configuration))
				.getClusterClient()) {

			ApplicationId applicationId = clusterClient.getClusterId();

			waitApplicationFinishedElseKillIt(
				applicationId, yarnAppTerminateTimeout, yarnClusterDescriptor, sleepIntervalInMS);
		}
	}
}
 
Example #2
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public ClusterClient<ApplicationId> deployJobCluster(
	ClusterSpecification clusterSpecification,
	JobGraph jobGraph,
	boolean detached) throws ClusterDeploymentException {

	// this is required because the slots are allocated lazily
	jobGraph.setAllowQueuedScheduling(true);

	try {
		return deployInternal(
			clusterSpecification,
			"Flink per-job cluster",
			getYarnJobClusterEntrypoint(),
			jobGraph,
			detached);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
	}
}
 
Example #3
Source File: KubernetesSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.set(DeploymentOptions.TARGET, KubernetesSessionClusterExecutor.NAME);
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	final Configuration executorConfig = cli.getEffectiveConfiguration(new String[]{});
	final ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example #4
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-ys", "3"};

	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCliWithJmAndTmTotalMemory(2048);

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
}
 
Example #5
Source File: KubernetesSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with unit (MB) for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] {
			"-e", KubernetesSessionClusterExecutor.NAME,
			"-D" + JobManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=1024m",
			"-D" + TaskManagerOptions.TOTAL_PROCESS_MEMORY.key() + "=2048m"
	};

	final KubernetesSessionCli cli = createFlinkKubernetesCustomCliWithJmAndTmTotalMemory(1024);

	final Configuration executorConfig = cli.getEffectiveConfiguration(args);
	final ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 
Example #6
Source File: KubernetesSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final String[] args = new String[] {
			"-e", KubernetesSessionClusterExecutor.NAME
	};

	final KubernetesSessionCli cli = createFlinkKubernetesCustomCliWithJmAndTmTotalMemory(1024);

	final Configuration executorConfig = cli.getEffectiveConfiguration(args);
	final ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
 
Example #7
Source File: KubernetesSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	final int taskManagerMemory = 7331;
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-e", KubernetesSessionClusterExecutor.NAME};
	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	Configuration executorConfig = cli.getEffectiveConfiguration(args);
	ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #8
Source File: Fabric8FlinkKubeClientTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
protected void onSetup() throws Exception {
	super.onSetup();

	KubernetesTestUtils.createTemporyFile("some data", flinkConfDir, "logback.xml");
	KubernetesTestUtils.createTemporyFile("some data", flinkConfDir, "log4j.properties");

	final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(JOB_MANAGER_MEMORY)
		.setTaskManagerMemoryMB(1000)
		.setSlotsPerTaskManager(3)
		.createClusterSpecification();

	final KubernetesJobManagerParameters kubernetesJobManagerParameters =
		new KubernetesJobManagerParameters(flinkConfig, clusterSpecification);
	this.kubernetesJobManagerSpecification =
		KubernetesJobManagerFactory.buildKubernetesJobManagerSpecification(kubernetesJobManagerParameters);
}
 
Example #9
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
 
Example #10
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #11
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory without unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 
Example #12
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
 
Example #13
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example #14
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example #15
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
 
Example #16
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public ClusterClientProvider<ApplicationId> deployJobCluster(
	ClusterSpecification clusterSpecification,
	JobGraph jobGraph,
	boolean detached) throws ClusterDeploymentException {
	try {
		return deployInternal(
			clusterSpecification,
			"Flink per-job cluster",
			getYarnJobClusterEntrypoint(),
			jobGraph,
			detached);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
	}
}
 
Example #17
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
 
Example #18
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #19
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example #20
Source File: AbstractJobClusterExecutor.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<JobClient> execute(@Nonnull final Pipeline pipeline, @Nonnull final Configuration configuration) throws Exception {
	final JobGraph jobGraph = PipelineExecutorUtils.getJobGraph(pipeline, configuration);

	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clusterClientFactory.createClusterDescriptor(configuration)) {
		final ExecutionConfigAccessor configAccessor = ExecutionConfigAccessor.fromConfiguration(configuration);

		final ClusterSpecification clusterSpecification = clusterClientFactory.getClusterSpecification(configuration);

		final ClusterClientProvider<ClusterID> clusterClientProvider = clusterDescriptor
				.deployJobCluster(clusterSpecification, jobGraph, configAccessor.getDetachedMode());
		LOG.info("Job has been submitted with JobID " + jobGraph.getJobID());

		return CompletableFuture.completedFuture(
				new ClusterClientJobClientAdapter<>(clusterClientProvider, jobGraph.getJobID()));
	}
}
 
Example #21
Source File: KubernetesClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ClusterClientProvider<String> deployJobCluster(
		ClusterSpecification clusterSpecification,
		JobGraph jobGraph,
		boolean detached) throws ClusterDeploymentException {
	throw new ClusterDeploymentException("Per-Job Mode not supported by Active Kubernetes deployments.");
}
 
Example #22
Source File: ApplicationClusterDeployer.java    From flink with Apache License 2.0 5 votes vote down vote up
public <ClusterID> void run(
		final Configuration configuration,
		final ApplicationConfiguration applicationConfiguration) throws Exception {
	checkNotNull(configuration);
	checkNotNull(applicationConfiguration);

	LOG.info("Submitting application in 'Application Mode'.");

	final ClusterClientFactory<ClusterID> clientFactory = clientServiceLoader.getClusterClientFactory(configuration);
	try (final ClusterDescriptor<ClusterID> clusterDescriptor = clientFactory.createClusterDescriptor(configuration)) {
		final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(configuration);

		clusterDescriptor.deployApplicationCluster(clusterSpecification, applicationConfiguration);
	}
}
 
Example #23
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests the specifying heap memory with unit (MB) for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithUnitMB() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024m", "-ytm", "2048m" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);
	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
 
Example #24
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
private void isReadyForDeployment(ClusterSpecification clusterSpecification) throws Exception {

		if (this.flinkJarPath == null) {
			throw new YarnDeploymentException("The Flink jar path is null");
		}
		if (this.flinkConfiguration == null) {
			throw new YarnDeploymentException("Flink configuration object has not been set");
		}

		// Check if we don't exceed YARN's maximum virtual cores.
		final int numYarnMaxVcores = yarnClusterInformationRetriever.getMaxVcores();

		int configuredAmVcores = flinkConfiguration.getInteger(YarnConfigOptions.APP_MASTER_VCORES);
		if (configuredAmVcores > numYarnMaxVcores) {
			throw new IllegalConfigurationException(
					String.format("The number of requested virtual cores for application master %d" +
									" exceeds the maximum number of virtual cores %d available in the Yarn Cluster.",
							configuredAmVcores, numYarnMaxVcores));
		}

		int configuredVcores = flinkConfiguration.getInteger(YarnConfigOptions.VCORES, clusterSpecification.getSlotsPerTaskManager());
		// don't configure more than the maximum configured number of vcores
		if (configuredVcores > numYarnMaxVcores) {
			throw new IllegalConfigurationException(
					String.format("The number of requested virtual cores per node %d" +
									" exceeds the maximum number of virtual cores %d available in the Yarn Cluster." +
									" Please note that the number of virtual cores is set to the number of task slots by default" +
									" unless configured in the Flink config with '%s.'",
							configuredVcores, numYarnMaxVcores, YarnConfigOptions.VCORES.key()));
		}

		// check if required Hadoop environment variables are set. If not, warn user
		if (System.getenv("HADOOP_CONF_DIR") == null &&
				System.getenv("YARN_CONF_DIR") == null) {
			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set. " +
					"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
					"configuration for accessing YARN.");
		}
	}
 
Example #25
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example #26
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ClusterClientProvider<ApplicationId> deploySessionCluster(ClusterSpecification clusterSpecification) throws ClusterDeploymentException {
	try {
		return deployInternal(
				clusterSpecification,
				"Flink session cluster",
				getYarnSessionClusterEntrypoint(),
				null,
				false);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Couldn't deploy Yarn session cluster", e);
	}
}
 
Example #27
Source File: YarnClusterDescriptorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConfigOverwrite() throws ClusterDeploymentException {
	Configuration configuration = new Configuration();
	// overwrite vcores in config
	configuration.setInteger(YarnConfigOptions.VCORES, Integer.MAX_VALUE);
	configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = createYarnClusterDescriptor(configuration);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	// configure slots
	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(1)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}
 
Example #28
Source File: YarnClusterDescriptor.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public ClusterClientProvider<ApplicationId> deployApplicationCluster(
		final ClusterSpecification clusterSpecification,
		final ApplicationConfiguration applicationConfiguration) throws ClusterDeploymentException {
	checkNotNull(clusterSpecification);
	checkNotNull(applicationConfiguration);

	final YarnDeploymentTarget deploymentTarget = YarnDeploymentTarget.fromConfig(flinkConfiguration);
	if (YarnDeploymentTarget.APPLICATION != deploymentTarget) {
		throw new ClusterDeploymentException(
				"Couldn't deploy Yarn Application Cluster." +
						" Expected deployment.target=" + YarnDeploymentTarget.APPLICATION.getName() +
						" but actual one was \"" + deploymentTarget.getName() + "\"");
	}

	applicationConfiguration.applyToConfiguration(flinkConfiguration);

	final List<String> pipelineJars = flinkConfiguration.getOptional(PipelineOptions.JARS).orElse(Collections.emptyList());
	Preconditions.checkArgument(pipelineJars.size() == 1, "Should only have one jar");

	try {
		return deployInternal(
				clusterSpecification,
				"Flink Application Cluster",
				YarnApplicationClusterEntryPoint.class.getName(),
				null,
				false);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Couldn't deploy Yarn Application Cluster", e);
	}
}
 
Example #29
Source File: KubernetesJobManagerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
protected void onSetup() throws Exception {
	final ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(JOB_MANAGER_MEMORY)
		.setTaskManagerMemoryMB(1024)
		.setSlotsPerTaskManager(3)
		.createClusterSpecification();

	this.kubernetesJobManagerParameters = new KubernetesJobManagerParameters(flinkConfig, clusterSpecification);

	this.baseFlinkPod = new FlinkPod.Builder().build();
}
 
Example #30
Source File: YarnClusterDescriptorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testFailIfTaskSlotsHigherThanMaxVcores() throws ClusterDeploymentException {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 0);

	YarnClusterDescriptor clusterDescriptor = createYarnClusterDescriptor(flinkConfiguration);

	clusterDescriptor.setLocalJarPath(new Path(flinkJar.getPath()));

	ClusterSpecification clusterSpecification = new ClusterSpecification.ClusterSpecificationBuilder()
		.setMasterMemoryMB(1)
		.setTaskManagerMemoryMB(1)
		.setNumberTaskManagers(1)
		.setSlotsPerTaskManager(Integer.MAX_VALUE)
		.createClusterSpecification();

	try {
		clusterDescriptor.deploySessionCluster(clusterSpecification);

		fail("The deploy call should have failed.");
	} catch (ClusterDeploymentException e) {
		// we expect the cause to be an IllegalConfigurationException
		if (!(e.getCause() instanceof IllegalConfigurationException)) {
			throw e;
		}
	} finally {
		clusterDescriptor.close();
	}
}