Java Code Examples for org.apache.flink.configuration.JobManagerOptions

The following examples show how to use org.apache.flink.configuration.JobManagerOptions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the command line arguments override the configuration settings
 * when the {@link ClusterSpecification} is created.
 */
@Test
public void testCommandLineClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	final int taskManagerMemory = 7331;
	final int slotsPerTaskManager = 30;

	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(jobManagerMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(taskManagerMemory));
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {"-yjm", String.valueOf(jobManagerMemory) + "m", "-ytm", String.valueOf(taskManagerMemory) + "m", "-ys", String.valueOf(slotsPerTaskManager)};
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 2
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 3
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example 4
Source Project: Flink-CEPplus   Source File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 
Example 5
Source Project: Flink-CEPplus   Source File: JobMasterConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public static JobMasterConfiguration fromConfiguration(Configuration configuration) {

		final Time rpcTimeout = AkkaUtils.getTimeoutAsTime(configuration);

		final Time slotRequestTimeout = Time.milliseconds(configuration.getLong(JobManagerOptions.SLOT_REQUEST_TIMEOUT));

		final String tmpDirectory = ConfigurationUtils.parseTempDirectories(configuration)[0];

		final RetryingRegistrationConfiguration retryingRegistrationConfiguration = RetryingRegistrationConfiguration.fromConfiguration(configuration);

		return new JobMasterConfiguration(
			rpcTimeout,
			slotRequestTimeout,
			tmpDirectory,
			retryingRegistrationConfiguration,
			configuration);
	}
 
Example 6
Source Project: flink   Source File: HistoryServerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	jmDirectory = tmpFolder.newFolder("jm_" + versionLessThan14);
	hsDirectory = tmpFolder.newFolder("hs_" + versionLessThan14);

	Configuration clusterConfig = new Configuration();
	clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(clusterConfig)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1)
			.build());
	cluster.before();
}
 
Example 7
Source Project: flink   Source File: FailoverStrategyLoader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Loads a FailoverStrategy Factory from the given configuration.
 */
public static FailoverStrategy.Factory loadFailoverStrategy(Configuration config, @Nullable Logger logger) {
	final String strategyParam = config.getString(
		JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
		FULL_RESTART_STRATEGY_NAME);

	if (StringUtils.isNullOrWhitespaceOnly(strategyParam)) {
		if (logger != null) {
			logger.warn("Null config value for {} ; using default failover strategy (full restarts).",
					JobManagerOptions.EXECUTION_FAILOVER_STRATEGY.key());
		}

		return new RestartAllStrategy.Factory();
	}
	else {
		switch (strategyParam.toLowerCase()) {
			case FULL_RESTART_STRATEGY_NAME:
				return new RestartAllStrategy.Factory();

			default:
				// we could interpret the parameter as a factory class name and instantiate that
				// for now we simply do not support this
				throw new IllegalConfigurationException("Unknown failover strategy: " + strategyParam);
		}
	}
}
 
Example 8
Source Project: flink   Source File: KubernetesSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.set(DeploymentOptions.TARGET, KubernetesSessionClusterExecutor.NAME);
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final KubernetesSessionCli cli = new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());

	final Configuration executorConfig = cli.getEffectiveConfiguration(new String[]{});
	final ClusterClientFactory<String> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example 9
Source Project: kylin   Source File: FlinkOnYarnConfigMappingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFlinkOnYarnJMMemOption() {
    String flinkonYarnJMMemOption = "-yjm";
    Map<String, String> map = FlinkOnYarnConfigMapping.flinkOnYarnConfigMap;
    for (Map.Entry<String, String> entry : map.entrySet()) {
        if (entry.getValue().equals(flinkonYarnJMMemOption)) {
            String flinkConfigOption = entry.getKey();

            boolean matchedAnyOne;
            matchedAnyOne = flinkConfigOption.equals(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.key());
            if (!matchedAnyOne) {
                if (JobManagerOptions.JOB_MANAGER_HEAP_MEMORY.hasFallbackKeys()) {
                    Iterator<FallbackKey> deprecatedKeyIterator = JobManagerOptions.JOB_MANAGER_HEAP_MEMORY
                            .fallbackKeys().iterator();
                    while (deprecatedKeyIterator.hasNext()) {
                        matchedAnyOne = matchedAnyOne && flinkConfigOption.equals(deprecatedKeyIterator.next().getKey());
                    }
                }
            }

            Assert.assertTrue(matchedAnyOne);
        }
    }
}
 
Example 10
Source Project: flink   Source File: DispatcherProcess.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Entrypoint of the DispatcherProcessEntryPoint.
 *
 * <p>Other arguments are parsed to a {@link Configuration} and passed to the Dispatcher,
 * for instance: <code>--high-availability ZOOKEEPER --high-availability.zookeeper.quorum
 * "xyz:123:456"</code>.
 */
public static void main(String[] args) {
	try {
		ParameterTool params = ParameterTool.fromArgs(args);
		Configuration config = params.getConfiguration();
		LOG.info("Configuration: {}.", config);

		config.setInteger(JobManagerOptions.PORT, 0);
		config.setString(RestOptions.BIND_PORT, "0");

		final StandaloneSessionClusterEntrypoint clusterEntrypoint = new StandaloneSessionClusterEntrypoint(config);

		ClusterEntrypoint.runClusterEntrypoint(clusterEntrypoint);
	}
	catch (Throwable t) {
		LOG.error("Failed to start Dispatcher process", t);
		System.exit(1);
	}
}
 
Example 11
Source Project: flink   Source File: FlinkILoopTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConfigurationForwardingStreamEnvironment() {
	Configuration configuration = new Configuration();
	configuration.setString("foobar", "foobar");
	configuration.setString(JobManagerOptions.ADDRESS, "localhost");
	configuration.setInteger(JobManagerOptions.PORT, 6123);

	FlinkILoop flinkILoop = new FlinkILoop(configuration, Option.<String[]>empty());

	StreamExecutionEnvironment streamEnv = flinkILoop.scalaSenv().getJavaEnv();
	assertTrue(streamEnv instanceof ScalaShellStreamEnvironment);

	ScalaShellStreamEnvironment remoteStreamEnv = (ScalaShellStreamEnvironment) streamEnv;
	Configuration forwardedConfiguration = remoteStreamEnv.getClientConfiguration();
	assertEquals(configuration, forwardedConfiguration);
}
 
Example 12
Source Project: flink   Source File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns the JobManager's hostname and port extracted from the given
 * {@link Configuration}.
 *
 * @param configuration Configuration to extract the JobManager's address from
 * @return The JobManager's hostname and port
 * @throws ConfigurationException if the JobManager's address cannot be extracted from the configuration
 */
public static Tuple2<String, Integer> getJobManagerAddress(Configuration configuration) throws ConfigurationException {

	final String hostname = configuration.getString(JobManagerOptions.ADDRESS);
	final int port = configuration.getInteger(JobManagerOptions.PORT);

	if (hostname == null) {
		throw new ConfigurationException("Config parameter '" + JobManagerOptions.ADDRESS +
			"' is missing (hostname/address of JobManager to connect to).");
	}

	if (port <= 0 || port >= 65536) {
		throw new ConfigurationException("Invalid value for '" + JobManagerOptions.PORT +
			"' (port of the JobManager actor system) : " + port +
			".  it must be greater than 0 and less than 65536.");
	}

	return Tuple2.of(hostname, port);
}
 
Example 13
Source Project: flink   Source File: ExecutionJobVertex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Convenience constructor for testing.
 */
@VisibleForTesting
ExecutionJobVertex(
		ExecutionGraph graph,
		JobVertex jobVertex,
		int defaultParallelism,
		Time timeout) throws JobException {

	this(
		graph,
		jobVertex,
		defaultParallelism,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		timeout,
		1L,
		System.currentTimeMillis());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: ClientTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {

	ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment();
	env.generateSequence(1, 1000).output(new DiscardingOutputFormat<Long>());

	Plan plan = env.createProgramPlan();
	JobWithJars jobWithJars = new JobWithJars(plan, Collections.<URL>emptyList(),  Collections.<URL>emptyList());

	program = mock(PackagedProgram.class);
	when(program.getPlanWithJars()).thenReturn(jobWithJars);

	final int freePort = NetUtils.getAvailablePort();
	config = new Configuration();
	config.setString(JobManagerOptions.ADDRESS, "localhost");
	config.setInteger(JobManagerOptions.PORT, freePort);
	config.setString(AkkaOptions.ASK_TIMEOUT, AkkaOptions.ASK_TIMEOUT.defaultValue());
}
 
Example 15
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests the specifying heap memory with old config key for job manager and task manager.
 */
@Test
public void testHeapMemoryPropertyWithOldConfigKey() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY_MB, 2048);
	configuration.setInteger(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY_MB, 4096);

	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli(configuration);

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[0], false);

	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);
	final ClusterSpecification clusterSpecification = clientFactory.getClusterSpecification(executorConfig);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(2048));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(4096));
}
 
Example 16
Source Project: flink   Source File: MesosSessionClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void initializeServices(Configuration config) throws Exception {
	super.initializeServices(config);

	final String hostname = config.getString(JobManagerOptions.ADDRESS);

	// Mesos configuration
	mesosConfig = MesosEntrypointUtils.createMesosSchedulerConfiguration(config, hostname);

	// services
	mesosServices = MesosServicesUtils.createMesosServices(config, hostname);

	// TM configuration
	taskManagerParameters = MesosEntrypointUtils.createTmParameters(config, LOG);
	taskManagerContainerSpec = MesosEntrypointUtils.createContainerSpec(config, dynamicProperties);
}
 
Example 17
Source Project: flink   Source File: KubernetesEntrypointUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * For non-HA cluster, {@link JobManagerOptions#ADDRESS} has be set to Kubernetes service name on client side. See
 * {@link KubernetesClusterDescriptor#deployClusterInternal}. So the TaskManager will use service address to contact
 * with JobManager.
 * For HA cluster, {@link JobManagerOptions#ADDRESS} will be set to the pod ip address. The TaskManager use Zookeeper
 * or other high-availability service to find the address of JobManager.
 *
 * @return Updated configuration
 */
static Configuration loadConfiguration() {
	final String configDir = System.getenv(ConfigConstants.ENV_FLINK_CONF_DIR);
	Preconditions.checkNotNull(
		configDir,
		"Flink configuration directory (%s) in environment should not be null!",
		ConfigConstants.ENV_FLINK_CONF_DIR);

	final Configuration configuration = GlobalConfiguration.loadConfiguration(configDir);

	if (HighAvailabilityMode.isHighAvailabilityModeActivated(configuration)) {
		final String ipAddress = System.getenv().get(Constants.ENV_FLINK_POD_IP_ADDRESS);
		Preconditions.checkState(
			ipAddress != null,
			"JobManager ip address environment variable %s not set",
			Constants.ENV_FLINK_POD_IP_ADDRESS);
		configuration.setString(JobManagerOptions.ADDRESS, ipAddress);
		configuration.setString(RestOptions.ADDRESS, ipAddress);
	}

	return configuration;
}
 
Example 18
Source Project: flink   Source File: HistoryServerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	jmDirectory = TMP.newFolder("jm_" + versionLessThan14);
	hsDirectory = TMP.newFolder("hs_" + versionLessThan14);

	Configuration clusterConfig = new Configuration();
	clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(clusterConfig)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1)
			.build());
	cluster.before();
}
 
Example 19
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 20
Source Project: flink   Source File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 
Example 21
Source Project: flink   Source File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(768));
	flinkConfiguration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.parse("1g"));
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 20000);

	flinkConfiguration.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 
Example 22
Source Project: flink   Source File: ClusterEntrypoint.java    License: Apache License 2.0 6 votes vote down vote up
protected static Configuration loadConfiguration(EntrypointClusterConfiguration entrypointClusterConfiguration) {
	final Configuration dynamicProperties = ConfigurationUtils.createConfiguration(entrypointClusterConfiguration.getDynamicProperties());
	final Configuration configuration = GlobalConfiguration.loadConfiguration(entrypointClusterConfiguration.getConfigDir(), dynamicProperties);

	final int restPort = entrypointClusterConfiguration.getRestPort();

	if (restPort >= 0) {
		configuration.setInteger(RestOptions.PORT, restPort);
	}

	final String hostname = entrypointClusterConfiguration.getHostname();

	if (hostname != null) {
		configuration.setString(JobManagerOptions.ADDRESS, hostname);
	}

	return configuration;
}
 
Example 23
Source Project: flink   Source File: ExecutionJobVertex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Convenience constructor for testing.
 */
@VisibleForTesting
ExecutionJobVertex(
		ExecutionGraph graph,
		JobVertex jobVertex,
		int defaultParallelism,
		Time timeout) throws JobException {

	this(
		graph,
		jobVertex,
		defaultParallelism,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		timeout,
		1L,
		System.currentTimeMillis());
}
 
Example 24
Source Project: flink   Source File: KubernetesSessionCliTest.java    License: Apache License 2.0 5 votes vote down vote up
private KubernetesSessionCli createFlinkKubernetesCustomCliWithJmAndTmTotalMemory(int totalMemory) {
	Configuration configuration = new Configuration();
	configuration.set(JobManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(totalMemory));
	configuration.set(TaskManagerOptions.TOTAL_PROCESS_MEMORY, MemorySize.ofMebiBytes(totalMemory));
	return new KubernetesSessionCli(
			configuration,
			tmp.getRoot().getAbsolutePath());
}
 
Example 25
Source Project: flink   Source File: SchedulerNGFactoryFactoryTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void throwsExceptionIfSchedulerNameIsInvalid() {
	final Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.SCHEDULER, "invalid-scheduler-name");

	try {
		createSchedulerNGFactory(configuration);
	} catch (IllegalArgumentException e) {
		assertThat(e.getMessage(), containsString("Illegal value [invalid-scheduler-name]"));
	}
}
 
Example 26
Source Project: flink   Source File: RestClusterClientTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that command line options override the configuration settings.
 */
@Test
public void testRESTManualConfigurationOverride() throws Exception {
	final String configuredHostname = "localhost";
	final int configuredPort = 1234;
	final Configuration configuration = new Configuration();

	configuration.setString(JobManagerOptions.ADDRESS, configuredHostname);
	configuration.setInteger(JobManagerOptions.PORT, configuredPort);
	configuration.setString(RestOptions.ADDRESS, configuredHostname);
	configuration.setInteger(RestOptions.PORT, configuredPort);

	final DefaultCLI defaultCLI = new DefaultCLI(configuration);

	final String manualHostname = "123.123.123.123";
	final int manualPort = 4321;
	final String[] args = {"-m", manualHostname + ':' + manualPort};

	CommandLine commandLine = defaultCLI.parseCommandLineOptions(args, false);

	final ClusterClientServiceLoader serviceLoader = new DefaultClusterClientServiceLoader();
	final Configuration executorConfig = defaultCLI.applyCommandLineOptionsToConfiguration(commandLine);

	final ClusterClientFactory<StandaloneClusterId> clusterFactory = serviceLoader.getClusterClientFactory(executorConfig);
	checkState(clusterFactory != null);

	final ClusterDescriptor<StandaloneClusterId> clusterDescriptor = clusterFactory.createClusterDescriptor(executorConfig);
	final RestClusterClient<?> clusterClient = (RestClusterClient<?>) clusterDescriptor
			.retrieve(clusterFactory.getClusterId(executorConfig))
			.getClusterClient();

	URL webMonitorBaseUrl = clusterClient.getWebMonitorBaseUrl().get();
	assertThat(webMonitorBaseUrl.getHost(), equalTo(manualHostname));
	assertThat(webMonitorBaseUrl.getPort(), equalTo(manualPort));
}
 
Example 27
Source Project: flink   Source File: JobManagerFlinkMemoryUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public JobManagerFlinkMemory deriveFromTotalFlinkMemory(Configuration config, MemorySize totalFlinkMemorySize) {
	MemorySize offHeapMemorySize = ProcessMemoryUtils.getMemorySizeFromConfig(config, JobManagerOptions.OFF_HEAP_MEMORY);
	if (totalFlinkMemorySize.compareTo(offHeapMemorySize) < 1) {
		throw new IllegalConfigurationException(
			"The configured Total Flink Memory (%s) is less than the configured Off-heap Memory (%s).",
			totalFlinkMemorySize.toHumanReadableString(),
			offHeapMemorySize.toHumanReadableString());
	}
	MemorySize derivedJvmHeapMemorySize = totalFlinkMemorySize.subtract(offHeapMemorySize);
	return createJobManagerFlinkMemory(derivedJvmHeapMemorySize, offHeapMemorySize);
}
 
Example 28
Source Project: flink-statefun   Source File: Harness.java    License: Apache License 2.0 5 votes vote down vote up
private static void configureStrictlyRequiredFlinkConfigs(Configuration flinkConfig) {
  flinkConfig.set(
      CoreOptions.ALWAYS_PARENT_FIRST_LOADER_PATTERNS_ADDITIONAL,
      String.join(";", StatefulFunctionsConfigValidator.PARENT_FIRST_CLASSLOADER_PATTERNS));
  flinkConfig.set(
      ExecutionCheckpointingOptions.MAX_CONCURRENT_CHECKPOINTS,
      StatefulFunctionsConfigValidator.MAX_CONCURRENT_CHECKPOINTS);
  flinkConfig.set(JobManagerOptions.SCHEDULER, SchedulerNGFactoryFactory.SCHEDULER_TYPE_LEGACY);
}
 
Example 29
Source Project: flink   Source File: CliFrontend.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes the given job manager address to the associated configuration object.
 *
 * @param address Address to write to the configuration
 * @param config The configuration to write to
 */
static void setJobManagerAddressInConfig(Configuration config, InetSocketAddress address) {
	config.setString(JobManagerOptions.ADDRESS, address.getHostString());
	config.setInteger(JobManagerOptions.PORT, address.getPort());
	config.setString(RestOptions.ADDRESS, address.getHostString());
	config.setInteger(RestOptions.PORT, address.getPort());
}
 
Example 30
Source Project: flink   Source File: HistoryServerArchivist.java    License: Apache License 2.0 5 votes vote down vote up
static HistoryServerArchivist createHistoryServerArchivist(Configuration configuration, JsonArchivist jsonArchivist, Executor ioExecutor) {
	final String configuredArchivePath = configuration.getString(JobManagerOptions.ARCHIVE_DIR);

	if (configuredArchivePath != null) {
		final Path archivePath = WebMonitorUtils.validateAndNormalizeUri(new Path(configuredArchivePath).toUri());

		return new JsonResponseHistoryServerArchivist(jsonArchivist, archivePath, ioExecutor);
	} else {
		return VoidHistoryServerArchivist.INSTANCE;
	}
}