org.apache.flink.yarn.cli.FlinkYarnSessionCli Java Examples

The following examples show how to use org.apache.flink.yarn.cli.FlinkYarnSessionCli. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testZookeeperNamespaceProperty() throws Exception {
	String zkNamespaceCliInput = "flink_test_namespace";

	String[] params = new String[] {"-yn", "2", "-yz", zkNamespaceCliInput};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(zkNamespaceCliInput, descriptor.getZookeeperNamespace());
}
Example #2
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	assertEquals(2, flinkYarnDescriptor.shipFiles.size());

}
Example #3
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
Example #4
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory without unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example #5
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example #6
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
Example #7
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testYarnIDOverridesPropertiesFile() throws Exception {
	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true);
	final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);
	assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId);
}
Example #8
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example #9
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example #10
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] { "-yn", "2", "-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
Example #11
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that the CliFrontend is able to pick up the .yarn-properties file from a specified location.
 */
@Test
public void testResumeFromYarnPropertiesFile() throws Exception {

	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {}, true);

	final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);

	assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
Example #12
Source File: FlinkYarnSessionCliTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example #13
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example #14
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-yn", "2", "-ys", "3"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	final ClusterSpecification clusterSpecification = yarnCLI.getClusterSpecification(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
	assertEquals(2, clusterSpecification.getNumberTaskManagers());
}
Example #15
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfDetachedMode() throws Exception {
	String[] params =
		new String[] {"-yd"};

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertTrue(descriptor.isDetachedMode());
}
Example #16
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	final int taskManagerMemory = 7331;
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example #17
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testNodeLabelProperty() throws Exception {
	String nodeLabelCliInput = "flink_test_nodelabel";

	String[] params = new String[] { "-yn", "2", "-ynl", nodeLabelCliInput };

	FlinkYarnSessionCli yarnCLI = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	AbstractYarnClusterDescriptor descriptor = yarnCLI.createClusterDescriptor(commandLine);

	assertEquals(nodeLabelCliInput, descriptor.getNodeLabel());
}
Example #18
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Test that the CliFrontend is able to pick up the .yarn-properties file from a specified location.
 */
@Test
public void testResumeFromYarnPropertiesFile() throws Exception {

	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {}, true);

	final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);

	assertEquals(TEST_YARN_APPLICATION_ID, clusterId);
}
Example #19
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testYarnIDOverridesPropertiesFile() throws Exception {
	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ApplicationId clusterId = clientFactory.getClusterId(executorConfig);

	assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId);
}
Example #20
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example #21
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespace() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString()}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	String zkNs = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertTrue(zkNs.matches("application_\\d+_0042"));
}
Example #22
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
Example #23
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testYarnIDOverridesPropertiesFile() throws Exception {
	File directoryPath = writeYarnPropertiesFile(validPropertiesFile);

	final Configuration configuration = new Configuration();
	configuration.setString(YarnConfigOptions.PROPERTIES_FILE_LOCATION, directoryPath.getAbsolutePath());

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");
	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID_2.toString() }, true);
	final ApplicationId clusterId = flinkYarnSessionCli.getClusterId(commandLine);
	assertEquals(TEST_YARN_APPLICATION_ID_2, clusterId);
}
Example #24
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
Example #25
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory without unit for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithoutUnit() throws Exception {
	final String[] args = new String[] { "-yn", "2", "-yjm", "1024", "-ytm", "2048" };
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(2048));
}
Example #26
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
Example #27
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests the specifying heap memory with config default value for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithConfigDefaultValue() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(1024));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(1024));
}
Example #28
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testMultipleYarnShipOptions() throws Exception {
	final String[] args = new String[]{"run", "--yarnship", tmp.newFolder().getAbsolutePath(), "--yarnship", tmp.newFolder().getAbsolutePath()};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	AbstractYarnClusterDescriptor flinkYarnDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	assertEquals(2, flinkYarnDescriptor.shipFiles.size());

}
Example #29
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testCorrectSettingOfMaxSlots() throws Exception {
	String[] params =
		new String[] {"-ys", "3"};

	FlinkYarnSessionCli yarnCLI = createFlinkYarnSessionCliWithJmAndTmTotalMemory(2048);

	final CommandLine commandLine = yarnCLI.parseCommandLineOptions(params, true);

	final Configuration executorConfig = yarnCLI.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	// each task manager has 3 slots but the parallelism is 7. Thus the slots should be increased.
	assertEquals(3, clusterSpecification.getSlotsPerTaskManager());
}
Example #30
Source File: FlinkYarnSessionCliTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Test
public void testDynamicProperties() throws Exception {

	FlinkYarnSessionCli cli = new FlinkYarnSessionCli(
		new Configuration(),
		tmp.getRoot().getAbsolutePath(),
		"",
		"",
		false);
	Options options = new Options();
	cli.addGeneralOptions(options);
	cli.addRunOptions(options);

	CommandLineParser parser = new DefaultParser();
	CommandLine cmd = parser.parse(options, new String[]{"run", "-j", "fake.jar",
			"-D", AkkaOptions.ASK_TIMEOUT.key() + "=5 min",
			"-D", CoreOptions.FLINK_JVM_OPTIONS.key() + "=-DappName=foobar",
			"-D", SecurityOptions.SSL_INTERNAL_KEY_PASSWORD.key() + "=changeit"});

	Configuration executorConfig = cli.applyCommandLineOptionsToConfiguration(cmd);
	assertEquals("5 min", executorConfig.get(AkkaOptions.ASK_TIMEOUT));
	assertEquals("-DappName=foobar", executorConfig.get(CoreOptions.FLINK_JVM_OPTIONS));
	assertEquals("changeit", executorConfig.get(SecurityOptions.SSL_INTERNAL_KEY_PASSWORD));
}