Java Code Examples for org.apache.flink.configuration.TaskManagerOptions

The following examples show how to use org.apache.flink.configuration.TaskManagerOptions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: StreamTask.java    License: Apache License 2.0 6 votes vote down vote up
private void tryShutdownTimerService() {

		if (timerService != null && !timerService.isTerminated()) {

			try {
				final long timeoutMs = getEnvironment().getTaskManagerInfo().getConfiguration().
					getLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT_TIMERS);

				if (!timerService.shutdownServiceUninterruptible(timeoutMs)) {
					LOG.warn("Timer service shutdown exceeded time limit of {} ms while waiting for pending " +
						"timers. Will continue with shutdown procedure.", timeoutMs);
				}
			} catch (Throwable t) {
				// catch and log the exception to not replace the original exception
				LOG.error("Could not shut down timer service", t);
			}
		}
	}
 
Example 2
/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 
Example 3
Source Project: flink   Source File: TaskManagerRunnerConfigurationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreatingTaskManagerRpcServiceShouldFailIfRpcPortRangeIsInvalid() throws Exception {
	final Configuration config = new Configuration(createFlinkConfigWithPredefinedTaskManagerHostname("example.org"));
	config.setString(TaskManagerOptions.RPC_PORT, "-1");

	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	try {
		TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		fail("Should fail because -1 is not a valid port range");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(),  containsString("Invalid port range definition: -1"));
	} finally {
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example 4
@Test
public void testFlinkOnYarnTMMemOption() {
    String flinkonYarnTMMemOption = "-ytm";
    Map<String, String> map = FlinkOnYarnConfigMapping.flinkOnYarnConfigMap;
    for (Map.Entry<String, String> entry : map.entrySet()) {
        if (entry.getValue().equals(flinkonYarnTMMemOption)) {
            String flinkConfigOption = entry.getKey();

            boolean matchedAnyOne;
            matchedAnyOne = flinkConfigOption.equals(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.key());
            if (!matchedAnyOne) {
                if (TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY.hasFallbackKeys()) {
                    Iterator<FallbackKey> deprecatedKeyIterator = TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY
                            .fallbackKeys().iterator();
                    while (deprecatedKeyIterator.hasNext()) {
                        matchedAnyOne = matchedAnyOne && flinkConfigOption.equals(deprecatedKeyIterator.next().getKey());
                    }
                }
            }

            Assert.assertTrue(matchedAnyOne);
        }
    }
}
 
Example 5
Source Project: flink   Source File: ConfigurationParserUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses the configuration to get the page size and validates the value.
 *
 * @param configuration configuration object
 * @return size of memory segment
 */
public static int getPageSize(Configuration configuration) {
	final int pageSize = checkedDownCast(MemorySize.parse(
		configuration.getString(TaskManagerOptions.MEMORY_SEGMENT_SIZE)).getBytes());

	// check page size of for minimum size
	checkConfigParameter(
		pageSize >= MemoryManager.MIN_PAGE_SIZE,
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Minimum memory segment size is " + MemoryManager.MIN_PAGE_SIZE);
	// check page size for power of two
	checkConfigParameter(
		MathUtils.isPowerOf2(pageSize),
		pageSize,
		TaskManagerOptions.MEMORY_SEGMENT_SIZE.key(),
		"Memory segment size must be a power of 2.");

	return pageSize;
}
 
Example 6
protected Configuration createClusterConfig() throws IOException {
	TemporaryFolder temporaryFolder = new TemporaryFolder();
	temporaryFolder.create();
	final File haDir = temporaryFolder.newFolder();

	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m");
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");

	if (zkServer != null) {
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}
	return config;
}
 
Example 7
Source Project: flink   Source File: TaskManagerRunner.java    License: Apache License 2.0 6 votes vote down vote up
private static String determineTaskManagerBindAddressByConnectingToResourceManager(
		final Configuration configuration,
		final HighAvailabilityServices haServices) throws LeaderRetrievalException {

	final Time lookupTimeout = Time.milliseconds(AkkaUtils.getLookupTimeout(configuration).toMillis());

	final InetAddress taskManagerAddress = LeaderRetrievalUtils.findConnectingAddress(
		haServices.getResourceManagerLeaderRetriever(),
		lookupTimeout);

	LOG.info("TaskManager will use hostname/address '{}' ({}) for communication.",
		taskManagerAddress.getHostName(), taskManagerAddress.getHostAddress());

	HostBindPolicy bindPolicy = HostBindPolicy.fromString(configuration.getString(TaskManagerOptions.HOST_BIND_POLICY));
	return bindPolicy == HostBindPolicy.IP ? taskManagerAddress.getHostAddress() : taskManagerAddress.getHostName();
}
 
Example 8
Source Project: Flink-CEPplus   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 9
Source Project: Flink-CEPplus   Source File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 
Example 10
Source Project: flink   Source File: TestingMiniClusterConfiguration.java    License: Apache License 2.0 6 votes vote down vote up
public TestingMiniClusterConfiguration build() {
	final Configuration modifiedConfiguration = new Configuration(configuration);
	modifiedConfiguration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlotsPerTaskManager);
	modifiedConfiguration.setString(
		RestOptions.ADDRESS,
		modifiedConfiguration.getString(RestOptions.ADDRESS, "localhost"));
	modifiedConfiguration.setInteger(
		RestOptions.PORT,
		modifiedConfiguration.getInteger(RestOptions.PORT, 0));

	return new TestingMiniClusterConfiguration(
		modifiedConfiguration,
		numTaskManagers,
		rpcServiceSharing,
		commonBindAddress,
		numberDispatcherResourceManagerComponents,
		localCommunication);
}
 
Example 11
Source Project: flink   Source File: HAQueryableStateFsBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example 12
/**
 * Returns a flink configuration object with the given values.
 *
 * @param javaMemMB
 * 		total JVM memory to use (in megabytes)
 * @param useOffHeap
 * 		whether to use off-heap memory (<tt>true</tt>) or not (<tt>false</tt>)
 * @param netBufMemFrac
 * 		fraction of JVM memory to use for network buffers
 * @param netBufMemMin
 * 		minimum memory size for network buffers (in bytes)
 * @param netBufMemMax
 * 		maximum memory size for network buffers (in bytes)
 * @param managedMemSizeMB
 * 		amount of managed memory (in megabytes)
 * @param managedMemFrac
 * 		fraction of free memory to use for managed memory (if <tt>managedMemSizeMB</tt> is
 * 		<tt>-1</tt>)
 *
 * @return flink configuration
 */
private static Configuration getConfig(
		final int javaMemMB, final boolean useOffHeap, final float netBufMemFrac,
		final long netBufMemMin, final long netBufMemMax, final int managedMemSizeMB,
		final float managedMemFrac) {

	Configuration config = new Configuration();

	config.setLong(KEY_TASKM_MEM_SIZE, javaMemMB);
	config.setBoolean(TaskManagerOptions.MEMORY_OFF_HEAP, useOffHeap);

	config.setFloat(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_FRACTION, netBufMemFrac);
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MIN, String.valueOf(netBufMemMin));
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(netBufMemMax));

	if (managedMemSizeMB == 0) {
		config.removeConfig(TaskManagerOptions.MANAGED_MEMORY_SIZE);
	} else {
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, managedMemSizeMB + "m");
	}
	config.setFloat(TaskManagerOptions.MANAGED_MEMORY_FRACTION, managedMemFrac);

	return config;
}
 
Example 13
/**
 * Validates the (new) network buffer configuration.
 *
 * @param pageSize 				size of memory buffers
 * @param networkBufFraction	fraction of JVM memory to use for network buffers
 * @param networkBufMin 		minimum memory size for network buffers (in bytes)
 * @param networkBufMax 		maximum memory size for network buffers (in bytes)
 *
 * @throws IllegalConfigurationException if the condition does not hold
 */
protected static void checkNetworkBufferConfig(
		final int pageSize, final float networkBufFraction, final long networkBufMin,
		final long networkBufMax) throws IllegalConfigurationException {

	checkConfigParameter(networkBufFraction > 0.0f && networkBufFraction < 1.0f, networkBufFraction,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.key(),
		"Network buffer memory fraction of the free memory must be between 0.0 and 1.0");

	checkConfigParameter(networkBufMin >= pageSize, networkBufMin,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.key(),
		"Minimum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= pageSize, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must allow at least one network " +
			"buffer with respect to the memory segment size");

	checkConfigParameter(networkBufMax >= networkBufMin, networkBufMax,
		TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key(),
		"Maximum memory for network buffers must not be smaller than minimum memory (" +
			TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.key() + ": " + networkBufMin + ")");
}
 
Example 14
Source Project: flink   Source File: EventTimeWindowCheckpointingITCase.java    License: Apache License 2.0 6 votes vote down vote up
protected Configuration createClusterConfig() throws IOException {
	TemporaryFolder temporaryFolder = new TemporaryFolder();
	temporaryFolder.create();
	final File haDir = temporaryFolder.newFolder();

	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "48m");
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(NettyShuffleEnvironmentOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	config.setString(AkkaOptions.FRAMESIZE, String.valueOf(MAX_MEM_STATE_SIZE) + "b");

	if (zkServer != null) {
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}
	return config;
}
 
Example 15
Source Project: flink   Source File: ConfigurationParserUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Parses the configuration to get the managed memory size and validates the value.
 *
 * @param configuration configuration object
 * @return managed memory size (in megabytes)
 */
public static long getManagedMemorySize(Configuration configuration) {
	long managedMemorySize;
	String managedMemorySizeDefaultVal = TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue();
	if (!configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(managedMemorySizeDefaultVal)) {
		try {
			managedMemorySize = MemorySize.parse(
				configuration.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE), MEGA_BYTES).getMebiBytes();
		} catch (IllegalArgumentException e) {
			throw new IllegalConfigurationException("Could not read " + TaskManagerOptions.MANAGED_MEMORY_SIZE.key(), e);
		}
	} else {
		managedMemorySize = Long.valueOf(managedMemorySizeDefaultVal);
	}

	checkConfigParameter(configuration.getString(
		TaskManagerOptions.MANAGED_MEMORY_SIZE).equals(TaskManagerOptions.MANAGED_MEMORY_SIZE.defaultValue()) || managedMemorySize > 0,
		managedMemorySize, TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
		"MemoryManager needs at least one MB of memory. " +
			"If you leave this config parameter empty, the system automatically pick a fraction of the available memory.");

	return managedMemorySize;
}
 
Example 16
Source Project: flink   Source File: BootstrapTools.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generate a task manager configuration.
 * @param baseConfig Config to start from.
 * @param jobManagerHostname Job manager host name.
 * @param jobManagerPort Port of the job manager.
 * @param numSlots Number of slots to configure.
 * @param registrationTimeout Timeout for registration
 * @return TaskManager configuration
 */
public static Configuration generateTaskManagerConfiguration(
			Configuration baseConfig,
			String jobManagerHostname,
			int jobManagerPort,
			int numSlots,
			FiniteDuration registrationTimeout) {

	Configuration cfg = cloneConfiguration(baseConfig);

	if (jobManagerHostname != null && !jobManagerHostname.isEmpty()) {
		cfg.setString(JobManagerOptions.ADDRESS, jobManagerHostname);
	}

	if (jobManagerPort > 0) {
		cfg.setInteger(JobManagerOptions.PORT, jobManagerPort);
	}

	cfg.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, registrationTimeout.toString());
	if (numSlots != -1){
		cfg.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, numSlots);
	}

	return cfg;
}
 
Example 17
Source Project: flink   Source File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the configuration settings are used to create the
 * {@link ClusterSpecification}.
 */
@Test
public void testConfigurationClusterSpecification() throws Exception {
	final Configuration configuration = new Configuration();
	final int jobManagerMemory = 1337;
	configuration.setString(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY, jobManagerMemory + "m");
	final int taskManagerMemory = 7331;
	configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, taskManagerMemory + "m");
	final int slotsPerTaskManager = 42;
	configuration.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, slotsPerTaskManager);

	final String[] args = {};
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(args, false);

	final ClusterSpecification clusterSpecification = flinkYarnSessionCli.getClusterSpecification(commandLine);

	assertThat(clusterSpecification.getMasterMemoryMB(), is(jobManagerMemory));
	assertThat(clusterSpecification.getTaskManagerMemoryMB(), is(taskManagerMemory));
	assertThat(clusterSpecification.getSlotsPerTaskManager(), is(slotsPerTaskManager));
}
 
Example 18
Source Project: Flink-CEPplus   Source File: LocalExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private JobExecutorService createJobExecutorService(Configuration configuration) throws Exception {
	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	final MiniClusterConfiguration miniClusterConfiguration = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumTaskManagers(
			configuration.getInteger(
				ConfigConstants.LOCAL_NUMBER_TASK_MANAGER,
				ConfigConstants.DEFAULT_LOCAL_NUMBER_TASK_MANAGER))
		.setRpcServiceSharing(RpcServiceSharing.SHARED)
		.setNumSlotsPerTaskManager(
			configuration.getInteger(
				TaskManagerOptions.NUM_TASK_SLOTS, 1))
		.build();

	final MiniCluster miniCluster = new MiniCluster(miniClusterConfiguration);
	miniCluster.start();

	configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

	return miniCluster;
}
 
Example 19
Source Project: Flink-CEPplus   Source File: TaskTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * The invoke() method holds a lock (trigger awaitLatch after acquisition)
 * and cancel cannot complete because it also tries to acquire the same lock.
 * This is resolved by the watch dog, no fatal error.
 */
@Test
public void testInterruptibleSharedLockInInvokeAndCancel() throws Exception {
	final TaskManagerActions taskManagerActions = new ProhibitFatalErrorTaskManagerActions();

	final Configuration config = new Configuration();
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_INTERVAL, 5);
	config.setLong(TaskManagerOptions.TASK_CANCELLATION_TIMEOUT, 50);

	final Task task = new TaskBuilder()
		.setInvokable(InvokableInterruptibleSharedLockInInvokeAndCancel.class)
		.setTaskManagerConfig(config)
		.setTaskManagerActions(taskManagerActions)
		.build();

	task.startTaskThread();

	awaitLatch.await();

	task.cancelExecution();
	task.getExecutingThread().join();
}
 
Example 20
Source Project: flink   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 21
Source Project: Flink-CEPplus   Source File: TaskManagerServicesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test for {@link TaskManagerServices#calculateNetworkBufferMemory(long, Configuration)} using new
 * configurations via {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_FRACTION},
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MIN} and
 * {@link TaskManagerOptions#NETWORK_BUFFERS_MEMORY_MAX}.
 */
@Test
public void calculateNetworkBufNew() throws Exception {
	Configuration config = new Configuration();

	// (1) defaults
	final Float defaultFrac = TaskManagerOptions.NETWORK_BUFFERS_MEMORY_FRACTION.defaultValue();
	final Long defaultMin = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MIN.defaultValue()).getBytes();
	final Long defaultMax = MemorySize.parse(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX.defaultValue()).getBytes();
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 20)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((64L << 20 + 1), config));
	assertEquals(enforceBounds((long) (defaultFrac * (10L << 30)), defaultMin, defaultMax),
		TaskManagerServices.calculateNetworkBufferMemory((10L << 30), config));

	calculateNetworkBufNew(config);
}
 
Example 22
Source Project: flink   Source File: TaskManagerServices.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Calculates the amount of heap memory to use (to set via <tt>-Xmx</tt> and <tt>-Xms</tt>)
 * based on the total memory to use and the given configuration parameters.
 *
 * @param totalJavaMemorySizeMB
 * 		overall available memory to use (heap and off-heap)
 * @param config
 * 		configuration object
 *
 * @return heap memory to use (in megabytes)
 */
public static long calculateHeapSizeMB(long totalJavaMemorySizeMB, Configuration config) {
	Preconditions.checkArgument(totalJavaMemorySizeMB > 0);

	// all values below here are in bytes

	final long totalProcessMemory = megabytesToBytes(totalJavaMemorySizeMB);
	final long networkReservedMemory = getReservedNetworkMemory(config, totalProcessMemory);
	final long heapAndManagedMemory = totalProcessMemory - networkReservedMemory;

	if (config.getBoolean(TaskManagerOptions.MEMORY_OFF_HEAP)) {
		final long managedMemorySize = getManagedMemoryFromHeapAndManaged(config, heapAndManagedMemory);

		ConfigurationParserUtils.checkConfigParameter(managedMemorySize < heapAndManagedMemory, managedMemorySize,
			TaskManagerOptions.MANAGED_MEMORY_SIZE.key(),
				"Managed memory size too large for " + (networkReservedMemory >> 20) +
					" MB network buffer memory and a total of " + totalJavaMemorySizeMB +
					" MB JVM memory");

		return bytesToMegabytes(heapAndManagedMemory - managedMemorySize);
	}
	else {
		return bytesToMegabytes(heapAndManagedMemory);
	}
}
 
Example 23
/**
 * Test for {@link NettyShuffleEnvironmentConfiguration#calculateNetworkBufferMemory(long, Configuration)} using old
 * configurations via {@link NettyShuffleEnvironmentOptions#NETWORK_NUM_BUFFERS}.
 */
@SuppressWarnings("deprecation")
@Test
public void calculateNetworkBufOld() {
	Configuration config = new Configuration();
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, 1);

	// note: actual network buffer memory size is independent of the totalJavaMemorySize
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(10L << 20, config));
	assertEquals(MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes(),
		NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(64L << 20, config));

	// test integer overflow in the memory size
	int numBuffers = (int) ((2L << 32) / MemorySize.parse(TaskManagerOptions.MEMORY_SEGMENT_SIZE.defaultValue()).getBytes()); // 2^33
	config.setInteger(NettyShuffleEnvironmentOptions.NETWORK_NUM_BUFFERS, numBuffers);
	assertEquals(2L << 32, NettyShuffleEnvironmentConfiguration.calculateNetworkBufferMemory(2L << 33, config));
}
 
Example 24
/**
 * Tests that the TaskManagerRunner startup fails if the network stack cannot be initialized.
 */
@Test
public void testStartupWhenNetworkStackFailsToInitialize() throws Exception {
	final ServerSocket blocker = new ServerSocket(0, 50, InetAddress.getByName(LOCAL_HOST));

	try {
		final Configuration cfg = new Configuration();
		cfg.setInteger(TaskManagerOptions.DATA_PORT, blocker.getLocalPort());

		startTaskManager(
			cfg,
			rpcService,
			highAvailabilityServices);

		fail("Should throw IOException when the network stack cannot be initialized.");
	} catch (IOException e) {
		// ignored
	} finally {
		IOUtils.closeQuietly(blocker);
	}
}
 
Example 25
Source Project: Flink-CEPplus   Source File: TaskExecutorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMaximumRegistrationDuration() throws Exception {
	configuration.setString(TaskManagerOptions.REGISTRATION_TIMEOUT, "10 ms");

	final TaskExecutor taskExecutor = createTaskExecutor(new TaskManagerServicesBuilder().build());

	taskExecutor.start();

	try {
		final Throwable error = testingFatalErrorHandler.getErrorFuture().get();
		assertThat(error, is(notNullValue()));
		assertThat(ExceptionUtils.stripExecutionException(error), instanceOf(RegistrationTimeoutException.class));

		testingFatalErrorHandler.clearError();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 26
Source Project: flink   Source File: ResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that RM and TM create the same slot resource profiles.
 */
@Test
public void testCreateWorkerSlotProfiles() {
	final Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "100m");
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, 5);

	final ResourceProfile rmCalculatedResourceProfile =
		ResourceManager.createWorkerSlotProfiles(config).iterator().next();

	final ResourceProfile tmCalculatedResourceProfile =
		TaskManagerServices.computeSlotResourceProfile(
			config.getInteger(TaskManagerOptions.NUM_TASK_SLOTS),
			MemorySize.parse(config.getString(TaskManagerOptions.MANAGED_MEMORY_SIZE)).getBytes());

	assertEquals(rmCalculatedResourceProfile, tmCalculatedResourceProfile);
}
 
Example 27
@Test
public void testCreatingTaskManagerRpcServiceShouldFailIfRpcPortRangeIsInvalid() throws Exception {
	final Configuration config = new Configuration(createFlinkConfigWithPredefinedTaskManagerHostname("example.org"));
	config.setString(TaskManagerOptions.RPC_PORT, "-1");

	final HighAvailabilityServices highAvailabilityServices = createHighAvailabilityServices(config);

	try {
		TaskManagerRunner.createRpcService(config, highAvailabilityServices);
		fail("Should fail because -1 is not a valid port range");
	} catch (final IllegalArgumentException e) {
		assertThat(e.getMessage(),  containsString("Invalid port range definition: -1"));
	} finally {
		highAvailabilityServices.closeAndCleanupAllData();
	}
}
 
Example 28
@Test
public void createResourceManager_WithLessMemoryThanContainerizedHeapCutoffMin_ShouldSucceed() throws Exception {
	final StandaloneResourceManagerFactory resourceManagerFactory = StandaloneResourceManagerFactory.INSTANCE;

	final TestingRpcService rpcService = new TestingRpcService();
	try {
		final Configuration configuration = new Configuration();
		configuration.setString(TaskManagerOptions.TASK_MANAGER_HEAP_MEMORY, new MemorySize(128 * 1024 * 1024).toString());
		configuration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, 600);

		final ResourceManager<ResourceID> ignored = resourceManagerFactory.createResourceManager(
			configuration,
			ResourceID.generate(),
			rpcService,
			new TestingHighAvailabilityServices(),
			new TestingHeartbeatServices(),
			NoOpMetricRegistry.INSTANCE,
			new TestingFatalErrorHandler(),
			new ClusterInformation("foobar", 1234),
			null,
			UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup());
	} finally {
		RpcUtils.terminateRpcService(rpcService, Time.seconds(10L));
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: HBaseConnectorITCase.java    License: Apache License 2.0 5 votes vote down vote up
public static void setAsContext() {
	Configuration config = new Configuration();
	// the default network buffers size (10% of heap max =~ 150MB) seems to much for this test case
	config.setString(TaskManagerOptions.NETWORK_BUFFERS_MEMORY_MAX, String.valueOf(80L << 20)); // 80 MB
	final LocalEnvironment le = new LocalEnvironment(config);

	initializeContextEnvironment(new ExecutionEnvironmentFactory() {
		@Override
		public ExecutionEnvironment createExecutionEnvironment() {
			return le;
		}
	});
}
 
Example 30
Source Project: Flink-CEPplus   Source File: KafkaTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected static Configuration getFlinkConfiguration() {
	Configuration flinkConfig = new Configuration();
	flinkConfig.setString(AkkaOptions.WATCH_HEARTBEAT_PAUSE, "5 s");
	flinkConfig.setString(AkkaOptions.WATCH_HEARTBEAT_INTERVAL, "1 s");
	flinkConfig.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "16m");
	flinkConfig.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");
	flinkConfig.setString(ConfigConstants.METRICS_REPORTER_PREFIX + "my_reporter." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX, JMXReporter.class.getName());
	return flinkConfig;
}