org.apache.flink.configuration.WebOptions Java Examples

The following examples show how to use org.apache.flink.configuration.WebOptions. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: HAQueryableStateFsBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #2

Source File: BackPressureStatsTrackerImplITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Before
public void setUp() throws Exception {
	networkBufferPool = new NetworkBufferPool(100, 8192);
	testBufferPool = networkBufferPool.createBufferPool(1, Integer.MAX_VALUE);

	final Configuration configuration = new Configuration();
	configuration.setInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES, BACKPRESSURE_NUM_SAMPLES);

	testingMiniCluster = new TestingMiniCluster(new TestingMiniClusterConfiguration.Builder()
		.setNumTaskManagers(JOB_PARALLELISM)
		.setConfiguration(configuration)
		.build());
	testingMiniCluster.start();

	dispatcherGateway = testingMiniCluster.getDispatcherGatewayFuture().get(TIMEOUT_SECONDS, TimeUnit.SECONDS);
}

Example #3

Source File: RestServerEndpointConfigurationTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testBasicMapping() throws ConfigurationException {
	Configuration originalConfig = new Configuration();
	originalConfig.setString(RestOptions.ADDRESS, ADDRESS);
	originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS);
	originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT);
	originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH);
	originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath());

	final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig);
	Assert.assertEquals(ADDRESS, result.getRestAddress());
	Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress());
	Assert.assertEquals(BIND_PORT, result.getRestBindPortRange());
	Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength());
	Assert.assertThat(
		result.getUploadDir().toAbsolutePath().toString(),
		containsString(temporaryFolder.getRoot().getAbsolutePath()));
}

Example #4

Source File: RestHandlerConfiguration.java From Flink-CEPplus with Apache License 2.0

6 votes

public static RestHandlerConfiguration fromConfiguration(Configuration configuration) {
	final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL);

	final int maxCheckpointStatisticCacheEntries = configuration.getInteger(WebOptions.CHECKPOINTS_HISTORY_SIZE);

	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));

	final String rootDir = "flink-web-ui";
	final File webUiDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir);

	return new RestHandlerConfiguration(
		refreshInterval,
		maxCheckpointStatisticCacheEntries,
		timeout,
		webUiDir);
}

Example #5

Source File: NonHAQueryableStateRocksDBBackendITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #6

Source File: WebMonitorUtils.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Finds the Flink log directory using log.file Java property that is set during startup.
 */
public static LogFileLocation find(Configuration config) {
	final String logEnv = "log.file";
	String logFilePath = System.getProperty(logEnv);

	if (logFilePath == null) {
		LOG.warn("Log file environment variable '{}' is not set.", logEnv);
		logFilePath = config.getString(WebOptions.LOG_PATH);
	}

	// not configured, cannot serve log files
	if (logFilePath == null || logFilePath.length() < 4) {
		LOG.warn("JobManager log files are unavailable in the web dashboard. " +
			"Log file location not found in environment variable '{}' or configuration key '{}'.",
			logEnv, WebOptions.LOG_PATH);
		return new LogFileLocation(null, null);
	}

	String outFilePath = logFilePath.substring(0, logFilePath.length() - 3).concat("out");

	LOG.info("Determined location of main cluster component log file: {}", logFilePath);
	LOG.info("Determined location of main cluster component stdout file: {}", outFilePath);

	return new LogFileLocation(resolveFileLocation(logFilePath), resolveFileLocation(outFilePath));
}

Example #7

Source File: WebFrontendITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m");
	config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	return config;
}

Example #8

Source File: MetricFetcherImpl.java From Flink-CEPplus with Apache License 2.0

6 votes

@Nonnull
public static <T extends RestfulGateway> MetricFetcherImpl<T> fromConfiguration(
		final Configuration configuration,
		final MetricQueryServiceRetriever metricQueryServiceRetriever,
		final GatewayRetriever<T> dispatcherGatewayRetriever,
		final ExecutorService executor) {
	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));
	final long updateInterval = configuration.getLong(MetricOptions.METRIC_FETCHER_UPDATE_INTERVAL);

	return new MetricFetcherImpl<>(
		dispatcherGatewayRetriever,
		metricQueryServiceRetriever,
		executor,
		timeout,
		updateInterval);
}

Example #9

Source File: HAQueryableStateRocksDBBackendITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #10

Source File: NonHAQueryableStateFsBackendITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #11

Source File: HAQueryableStateFsBackendITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #12

Source File: WebFrontendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "12m");
	config.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, true);

	return config;
}

Example #13

Source File: WebMonitorUtils.java From flink with Apache License 2.0

6 votes

/**
 * Finds the Flink log directory using log.file Java property that is set during startup.
 */
public static LogFileLocation find(Configuration config) {
	final String logEnv = "log.file";
	String logFilePath = System.getProperty(logEnv);

	if (logFilePath == null) {
		LOG.warn("Log file environment variable '{}' is not set.", logEnv);
		logFilePath = config.getString(WebOptions.LOG_PATH);
	}

	// not configured, cannot serve log files
	if (logFilePath == null || logFilePath.length() < 4) {
		LOG.warn("JobManager log files are unavailable in the web dashboard. " +
			"Log file location not found in environment variable '{}' or configuration key '{}'.",
			logEnv, WebOptions.LOG_PATH);
		return new LogFileLocation(null, null);
	}

	String outFilePath = logFilePath.substring(0, logFilePath.length() - 3).concat("out");

	LOG.info("Determined location of main cluster component log file: {}", logFilePath);
	LOG.info("Determined location of main cluster component stdout file: {}", outFilePath);

	return new LogFileLocation(resolveFileLocation(logFilePath), resolveFileLocation(outFilePath));
}

Example #14

Source File: NonHAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #15

Source File: RestHandlerConfiguration.java From flink with Apache License 2.0

6 votes

public static RestHandlerConfiguration fromConfiguration(Configuration configuration) {
	final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL);

	final int maxCheckpointStatisticCacheEntries = configuration.getInteger(WebOptions.CHECKPOINTS_HISTORY_SIZE);

	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));

	final String rootDir = "flink-web-ui";
	final File webUiDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir);

	return new RestHandlerConfiguration(
		refreshInterval,
		maxCheckpointStatisticCacheEntries,
		timeout,
		webUiDir);
}

Example #16

Source File: NonHAQueryableStateFsBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #17

Source File: HAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #18

Source File: NonHAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #19

Source File: RestServerEndpointConfigurationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testBasicMapping() throws ConfigurationException {
	Configuration originalConfig = new Configuration();
	originalConfig.setString(RestOptions.ADDRESS, ADDRESS);
	originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS);
	originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT);
	originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH);
	originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath());

	final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig);
	Assert.assertEquals(ADDRESS, result.getRestAddress());
	Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress());
	Assert.assertEquals(BIND_PORT, result.getRestBindPortRange());
	Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength());
	Assert.assertThat(
		result.getUploadDir().toAbsolutePath().toString(),
		containsString(temporaryFolder.getRoot().getAbsolutePath()));
}

Example #20

Source File: RestHandlerConfiguration.java From flink with Apache License 2.0

6 votes

public static RestHandlerConfiguration fromConfiguration(Configuration configuration) {
	final long refreshInterval = configuration.getLong(WebOptions.REFRESH_INTERVAL);

	final int maxCheckpointStatisticCacheEntries = configuration.getInteger(WebOptions.CHECKPOINTS_HISTORY_SIZE);

	final Time timeout = Time.milliseconds(configuration.getLong(WebOptions.TIMEOUT));

	final String rootDir = "flink-web-ui";
	final File webUiDir = new File(configuration.getString(WebOptions.TMP_DIR), rootDir);

	final boolean webSubmitEnabled = configuration.getBoolean(WebOptions.SUBMIT_ENABLE);

	return new RestHandlerConfiguration(
		refreshInterval,
		maxCheckpointStatisticCacheEntries,
		timeout,
		webUiDir,
		webSubmitEnabled);
}

Example #21

Source File: WebFrontendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getClusterConfiguration() {
	Configuration config = new Configuration();
	try {
		File logDir = File.createTempFile("TestBaseUtils-logdir", null);
		assertTrue("Unable to delete temp file", logDir.delete());
		assertTrue("Unable to create temp directory", logDir.mkdir());
		File logFile = new File(logDir, "jobmanager.log");
		File outFile = new File(logDir, "jobmanager.out");

		Files.createFile(logFile.toPath());
		Files.createFile(outFile.toPath());

		config.setString(WebOptions.LOG_PATH, logFile.getAbsolutePath());
		config.setString(ConfigConstants.TASK_MANAGER_LOG_PATH_KEY, logFile.getAbsolutePath());
	} catch (Exception e) {
		throw new AssertionError("Could not setup test.", e);
	}

	// !!DO NOT REMOVE!! next line is required for tests
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("12m"));

	return config;
}

Example #22

Source File: HAQueryableStateFsBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #23

Source File: NonHAQueryableStateFsBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 1);
	config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 1);
	config.setString(
		QueryableStateOptions.PROXY_PORT_RANGE,
		QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setString(
		QueryableStateOptions.SERVER_PORT_RANGE,
		QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_PORT_COUNT));
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #24

Source File: HAQueryableStateRocksDBBackendITCase.java From flink with Apache License 2.0

6 votes

private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}

Example #25

Source File: RestServerEndpointConfigurationTest.java From flink with Apache License 2.0

6 votes

@Test
public void testBasicMapping() throws ConfigurationException {
	Configuration originalConfig = new Configuration();
	originalConfig.setString(RestOptions.ADDRESS, ADDRESS);
	originalConfig.setString(RestOptions.BIND_ADDRESS, BIND_ADDRESS);
	originalConfig.setString(RestOptions.BIND_PORT, BIND_PORT);
	originalConfig.setInteger(RestOptions.SERVER_MAX_CONTENT_LENGTH, CONTENT_LENGTH);
	originalConfig.setString(WebOptions.TMP_DIR, temporaryFolder.getRoot().getAbsolutePath());

	final RestServerEndpointConfiguration result = RestServerEndpointConfiguration.fromConfiguration(originalConfig);
	Assert.assertEquals(ADDRESS, result.getRestAddress());
	Assert.assertEquals(BIND_ADDRESS, result.getRestBindAddress());
	Assert.assertEquals(BIND_PORT, result.getRestBindPortRange());
	Assert.assertEquals(CONTENT_LENGTH, result.getMaxContentLength());
	Assert.assertThat(
		result.getUploadDir().toAbsolutePath().toString(),
		containsString(temporaryFolder.getRoot().getAbsolutePath()));
}

Example #26

Source File: BackPressureITCase.java From flink with Apache License 2.0

5 votes

private static Configuration createBackPressureSamplingConfiguration() {
	final Configuration configuration = new Configuration();
	configuration.setInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL, 1000);
	configuration.setInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES, 1);
	configuration.setInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL, Integer.MAX_VALUE);
	return configuration;
}

Example #27

Source File: WebMonitorEndpointTest.java From flink with Apache License 2.0

5 votes

@Test
public void cleansUpExpiredExecutionGraphs() throws Exception {
	final Configuration configuration = new Configuration();
	configuration.setString(RestOptions.ADDRESS, "localhost");
	configuration.setLong(WebOptions.REFRESH_INTERVAL, 5L);
	final ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
	final long timeout = 10000L;

	final OneShotLatch cleanupLatch = new OneShotLatch();
	final TestingExecutionGraphCache executionGraphCache = TestingExecutionGraphCache.newBuilder()
		.setCleanupRunnable(cleanupLatch::trigger)
		.build();
	try (final WebMonitorEndpoint<RestfulGateway> webMonitorEndpoint = new WebMonitorEndpoint<>(
		RestServerEndpointConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		configuration,
		RestHandlerConfiguration.fromConfiguration(configuration),
		CompletableFuture::new,
		NoOpTransientBlobService.INSTANCE,
		executor,
		VoidMetricFetcher.INSTANCE,
		new TestingLeaderElectionService(),
		executionGraphCache,
		new TestingFatalErrorHandler())) {

		webMonitorEndpoint.start();

		// check that the cleanup will be triggered
		cleanupLatch.await(timeout, TimeUnit.MILLISECONDS);
	} finally {
		ExecutorUtils.gracefulShutdown(timeout, TimeUnit.MILLISECONDS, executor);
	}
}

Example #28

Source File: LocalExecutorITCase.java From flink with Apache License 2.0

5 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #29

Source File: LocalExecutorITCase.java From flink with Apache License 2.0

5 votes

private static Configuration getConfig() {
	Configuration config = new Configuration();
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
	config.setBoolean(WebOptions.SUBMIT_ENABLE, false);
	return config;
}

Example #30

Source File: BackPressureITCase.java From flink with Apache License 2.0

5 votes

private static Configuration createBackPressureSamplingConfiguration() {
	final Configuration configuration = new Configuration();
	configuration.setInteger(WebOptions.BACKPRESSURE_REFRESH_INTERVAL, 1000);
	configuration.setInteger(WebOptions.BACKPRESSURE_NUM_SAMPLES, 1);
	configuration.setInteger(WebOptions.BACKPRESSURE_CLEANUP_INTERVAL, Integer.MAX_VALUE);
	return configuration;
}