Java Code Examples for org.apache.flink.configuration.HighAvailabilityOptions

The following examples show how to use org.apache.flink.configuration.HighAvailabilityOptions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Flink-CEPplus   Author: ljygz   File: RollingSinkSecuredITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration startSecureFlinkClusterWithRecoveryModeEnabled() {
	try {
		LOG.info("Starting Flink and ZK in secure mode");

		dfs.mkdirs(new Path("/flink/checkpoints"));
		dfs.mkdirs(new Path("/flink/recovery"));

		final Configuration result = new Configuration();

		result.setBoolean(ConfigConstants.LOCAL_START_WEBSERVER, false);
		result.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, 3);
		result.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
		result.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
		result.setString(HighAvailabilityOptions.HA_ZOOKEEPER_CHECKPOINTS_PATH, hdfsURI + "/flink/checkpoints");
		result.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI + "/flink/recovery");
		result.setString("state.backend.fs.checkpointdir", hdfsURI + "/flink/checkpoints");

		SecureTestEnvironment.populateFlinkSecureConfigurations(result);

		return result;
	} catch (Exception e) {
		throw new RuntimeException(e);
	}
}
 
Example #2
Source Project: flink   Author: apache   File: ApplicationDispatcherBootstrapTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testJobIdDefaultsToZeroWithHa() throws Throwable {
	final Configuration configurationUnderTest = getConfiguration();
	configurationUnderTest.set(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name());

	final CompletableFuture<JobID> submittedJobId = new CompletableFuture<>();

	final TestingDispatcherGateway.Builder dispatcherBuilder = new TestingDispatcherGateway.Builder()
			.setSubmitFunction(jobGraph -> {
				submittedJobId.complete(jobGraph.getJobID());
				return CompletableFuture.completedFuture(Acknowledge.get());
			})
			.setRequestJobStatusFunction(jobId -> CompletableFuture.completedFuture(JobStatus.FINISHED))
			.setRequestJobResultFunction(jobId -> CompletableFuture.completedFuture(createSuccessfulJobResult(jobId)));

	final CompletableFuture<Void> applicationFuture =
			runApplication(dispatcherBuilder, configurationUnderTest, 1);

	applicationFuture.get(TIMEOUT_SECONDS, TimeUnit.SECONDS);

	assertThat(submittedJobId.get(TIMEOUT_SECONDS, TimeUnit.SECONDS), is(new JobID(0L, 0L)));
}
 
Example #3
Source Project: Flink-CEPplus   Author: ljygz   File: HDFSTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 
Example #4
Source Project: flink   Author: flink-tpc-ds   File: HighAvailabilityServicesUtils.java    License: Apache License 2.0 6 votes vote down vote up
private static HighAvailabilityServices createCustomHAServices(Configuration config, Executor executor) throws FlinkException {
	final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
	final String haServicesClassName = config.getString(HighAvailabilityOptions.HA_MODE);

	final HighAvailabilityServicesFactory highAvailabilityServicesFactory = InstantiationUtil.instantiate(
		haServicesClassName,
		HighAvailabilityServicesFactory.class,
		classLoader);

	try {
		return highAvailabilityServicesFactory.createHAServices(config, executor);
	} catch (Exception e) {
		throw new FlinkException(
			String.format(
				"Could not create the ha services from the instantiated HighAvailabilityServicesFactory %s.",
				haServicesClassName),
			e);
	}
}
 
Example #5
Source Project: flink   Author: apache   File: BlobCacheCorruptionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 *
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 * @param corruptOnHAStore
 * 		whether the file should be corrupt in the HA store (<tt>true</tt>, required
 * 		<tt>highAvailability</tt> to be set) or on the {@link BlobServer}'s local store
 * 		(<tt>false</tt>)
 */
private void testGetFailsFromCorruptFile(final JobID jobId, BlobKey.BlobType blobType,
		boolean corruptOnHAStore) throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(jobId, blobType, corruptOnHAStore, config,
			blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #6
Source Project: flink   Author: flink-tpc-ds   File: AbstractCustomCommandLine.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 
Example #7
Source Project: flink   Author: flink-tpc-ds   File: HAQueryableStateFsBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "4m");
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example #8
Source Project: flink   Author: flink-tpc-ds   File: HighAvailabilityModeTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests HA mode configuration.
 */
@Test
public void testFromConfig() throws Exception {
	Configuration config = new Configuration();

	// Check default
	assertEquals(DEFAULT_HA_MODE, HighAvailabilityMode.fromConfig(config));

	// Check not equals default
	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name().toLowerCase());
	assertEquals(HighAvailabilityMode.ZOOKEEPER, HighAvailabilityMode.fromConfig(config));

	// Check factory class
	config.setString(HighAvailabilityOptions.HA_MODE, "factory.class.FQN");
	assertEquals(HighAvailabilityMode.FACTORY_CLASS, HighAvailabilityMode.fromConfig(config));
}
 
Example #9
Source Project: Flink-CEPplus   Author: ljygz   File: ZooKeeperCompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #10
Source Project: Flink-CEPplus   Author: ljygz   File: ZooKeeperLeaderRetrievalTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void before() throws Exception {
	testingServer = new TestingServer();

	config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, testingServer.getConnectString());

	CuratorFramework client = ZooKeeperUtils.startCuratorFramework(config);

	highAvailabilityServices = new ZooKeeperHaServices(
		client,
		TestingUtils.defaultExecutor(),
		config,
		new VoidBlobStore());
}
 
Example #11
Source Project: flink   Author: apache   File: HighAvailabilityModeTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests HA mode configuration.
 */
@Test
public void testFromConfig() throws Exception {
	Configuration config = new Configuration();

	// Check default
	assertEquals(DEFAULT_HA_MODE, HighAvailabilityMode.fromConfig(config));

	// Check not equals default
	config.setString(HighAvailabilityOptions.HA_MODE, HighAvailabilityMode.ZOOKEEPER.name().toLowerCase());
	assertEquals(HighAvailabilityMode.ZOOKEEPER, HighAvailabilityMode.fromConfig(config));

	// Check factory class
	config.setString(HighAvailabilityOptions.HA_MODE, "factory.class.FQN");
	assertEquals(HighAvailabilityMode.FACTORY_CLASS, HighAvailabilityMode.fromConfig(config));
}
 
Example #12
Source Project: Flink-CEPplus   Author: ljygz   File: BlobServerCorruptionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 */
@Test
public void testGetFailsFromCorruptFile() throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #13
Source Project: flink   Author: flink-tpc-ds   File: BlobCacheCorruptionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 *
 * @param jobId
 * 		job ID or <tt>null</tt> if job-unrelated
 * @param blobType
 * 		whether the BLOB should become permanent or transient
 * @param corruptOnHAStore
 * 		whether the file should be corrupt in the HA store (<tt>true</tt>, required
 * 		<tt>highAvailability</tt> to be set) or on the {@link BlobServer}'s local store
 * 		(<tt>false</tt>)
 */
private void testGetFailsFromCorruptFile(final JobID jobId, BlobKey.BlobType blobType,
		boolean corruptOnHAStore) throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(jobId, blobType, corruptOnHAStore, config,
			blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #14
Source Project: Flink-CEPplus   Author: ljygz   File: BlobCacheRecoveryTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobCacheRecovery(config, blobStoreService);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #15
Source Project: flink   Author: apache   File: NotifyCheckpointAbortedITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 
Example #16
Source Project: Flink-CEPplus   Author: ljygz   File: BlobCacheRetriesTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * A test where the connection fails twice and then the get operation succeeds
 * (with high availability set, job-related job).
 */
@Test
public void testBlobFetchRetriesHa() throws IOException {
	final Configuration config = new Configuration();
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH,
		temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobFetchRetries(config, blobStoreService, new JobID(), PERMANENT_BLOB);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #17
Source Project: flink   Author: apache   File: HDFSTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 
Example #18
Source Project: flink   Author: apache   File: BlobCacheRecoveryTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testBlobCacheRecovery(config, blobStoreService);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #19
Source Project: Flink-CEPplus   Author: ljygz   File: HighAvailabilityServicesUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateCustomHAServices() throws Exception {
	Configuration config = new Configuration();

	HighAvailabilityServices haServices = Mockito.mock(HighAvailabilityServices.class);
	TestHAFactory.haServices = haServices;

	Executor executor = Mockito.mock(Executor.class);

	config.setString(HighAvailabilityOptions.HA_MODE, TestHAFactory.class.getName());

	// when
	HighAvailabilityServices actualHaServices = HighAvailabilityServicesUtils.createAvailableOrEmbeddedServices(config, executor);

	// then
	assertSame(haServices, actualHaServices);

	// when
	actualHaServices = HighAvailabilityServicesUtils.createHighAvailabilityServices(config, executor,
		HighAvailabilityServicesUtils.AddressResolution.NO_ADDRESS_RESOLUTION);
	// then
	assertSame(haServices, actualHaServices);
}
 
Example #20
Source Project: flink   Author: apache   File: HAQueryableStateRocksDBBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
private static Configuration getConfig() throws Exception {

		Configuration config = new Configuration();
		config.setBoolean(QueryableStateOptions.ENABLE_QUERYABLE_STATE_PROXY_SERVER, true);
		config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("4m"));
		config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
		config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
		config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);
		config.setInteger(QueryableStateOptions.CLIENT_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.PROXY_NETWORK_THREADS, 2);
		config.setInteger(QueryableStateOptions.SERVER_NETWORK_THREADS, 2);
		config.setString(
			QueryableStateOptions.PROXY_PORT_RANGE,
			QS_PROXY_PORT_RANGE_START + "-" + (QS_PROXY_PORT_RANGE_START + NUM_TMS));
		config.setString(
			QueryableStateOptions.SERVER_PORT_RANGE,
			QS_SERVER_PORT_RANGE_START + "-" + (QS_SERVER_PORT_RANGE_START + NUM_TMS));
		config.setBoolean(WebOptions.SUBMIT_ENABLE, false);

		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().toString());

		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
		config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

		return config;
	}
 
Example #21
Source Project: Flink-CEPplus   Author: ljygz   File: AbstractCustomCommandLine.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Override configuration settings by specified command line options.
 *
 * @param commandLine containing the overriding values
 * @return Effective configuration with the overridden configuration settings
 */
protected Configuration applyCommandLineOptionsToConfiguration(CommandLine commandLine) throws FlinkException {
	final Configuration resultingConfiguration = new Configuration(configuration);

	if (commandLine.hasOption(addressOption.getOpt())) {
		String addressWithPort = commandLine.getOptionValue(addressOption.getOpt());
		InetSocketAddress jobManagerAddress = ClientUtils.parseHostPortAddress(addressWithPort);
		setJobManagerAddressInConfig(resultingConfiguration, jobManagerAddress);
	}

	if (commandLine.hasOption(zookeeperNamespaceOption.getOpt())) {
		String zkNamespace = commandLine.getOptionValue(zookeeperNamespaceOption.getOpt());
		resultingConfiguration.setString(HighAvailabilityOptions.HA_CLUSTER_ID, zkNamespace);
	}

	return resultingConfiguration;
}
 
Example #22
Source Project: Flink-CEPplus   Author: ljygz   File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
private YarnClusterDescriptor setupYarnClusterDescriptor() {
	final Configuration flinkConfiguration = new Configuration();
	flinkConfiguration.setString(YarnConfigOptions.APPLICATION_ATTEMPTS, "10");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");
	flinkConfiguration.setString(HighAvailabilityOptions.HA_STORAGE_PATH, storageDir);
	flinkConfiguration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	flinkConfiguration.setInteger(HighAvailabilityOptions.ZOOKEEPER_SESSION_TIMEOUT, 1000);

	flinkConfiguration.setString(ConfigConstants.RESTART_STRATEGY, "fixed-delay");
	flinkConfiguration.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, Integer.MAX_VALUE);

	final int minMemory = 100;
	flinkConfiguration.setInteger(ResourceManagerOptions.CONTAINERIZED_HEAP_CUTOFF_MIN, minMemory);

	return createYarnClusterDescriptor(flinkConfiguration);
}
 
Example #23
Source Project: flink   Author: apache   File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final FlinkYarnSessionCli flinkYarnSessionCli = createFlinkYarnSessionCli();

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);
	final Configuration executorConfig = flinkYarnSessionCli.applyCommandLineOptionsToConfiguration(commandLine);
	final ClusterClientFactory<ApplicationId> clientFactory = getClusterClientFactory(executorConfig);

	final YarnClusterDescriptor clusterDescriptor = (YarnClusterDescriptor) clientFactory.createClusterDescriptor(executorConfig);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();
	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 
Example #24
Source Project: flink   Author: flink-tpc-ds   File: HDFSTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when talking to the {@link org.apache.flink.runtime.blob.BlobServer} directly.
 */
@Test
public void testBlobServerRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobServerRecoveryTest.testBlobServerRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 
Example #25
Source Project: flink   Author: flink-tpc-ds   File: HDFSTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed JARs are recoverable from any
 * participating BlobServer when uploaded via a BLOB cache.
 */
@Test
public void testBlobCacheRecovery() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheRecoveryTest.testBlobCacheRecovery(config, blobStoreService);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 
Example #26
Source Project: flink   Author: flink-tpc-ds   File: HDFSTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that with {@link HighAvailabilityMode#ZOOKEEPER} distributed corrupted JARs are
 * recognised during the download via a BLOB cache.
 */
@Test
public void testBlobCacheCorruptedFile() throws Exception {
	org.apache.flink.configuration.Configuration
		config = new org.apache.flink.configuration.Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, hdfsURI);

	BlobStoreService blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

	try {
		BlobCacheCorruptionTest
			.testGetFailsFromCorruptFile(new JobID(), config, blobStoreService, exception);
	} finally {
		blobStoreService.closeAndCleanupAllData();
	}
}
 
Example #27
Source Project: flink   Author: flink-tpc-ds   File: FlinkYarnSessionCliTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResumeFromYarnIDZookeeperNamespaceOverride() throws Exception {
	final Configuration configuration = new Configuration();
	final FlinkYarnSessionCli flinkYarnSessionCli = new FlinkYarnSessionCli(
		configuration,
		tmp.getRoot().getAbsolutePath(),
		"y",
		"yarn");

	final String overrideZkNamespace = "my_cluster";

	final CommandLine commandLine = flinkYarnSessionCli.parseCommandLineOptions(new String[] {"-yid", TEST_YARN_APPLICATION_ID.toString(), "-yz", overrideZkNamespace}, true);

	final AbstractYarnClusterDescriptor clusterDescriptor = flinkYarnSessionCli.createClusterDescriptor(commandLine);

	final Configuration clusterDescriptorConfiguration = clusterDescriptor.getFlinkConfiguration();

	final String clusterId = clusterDescriptorConfiguration.getValue(HighAvailabilityOptions.HA_CLUSTER_ID);
	assertEquals(overrideZkNamespace, clusterId);
}
 
Example #28
Source Project: flink   Author: flink-tpc-ds   File: ZooKeeperCompletedCheckpointStoreTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that checkpoints are discarded when the completed checkpoint store is shut
 * down with a globally terminal state.
 */
@Test
public void testDiscardingCheckpointsAtShutDown() throws Exception {
	final SharedStateRegistry sharedStateRegistry = new SharedStateRegistry();
	final Configuration configuration = new Configuration();
	configuration.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperResource.getConnectString());

	final CuratorFramework client = ZooKeeperUtils.startCuratorFramework(configuration);
	final ZooKeeperCompletedCheckpointStore checkpointStore = createZooKeeperCheckpointStore(client);

	try {
		final CompletedCheckpointStoreTest.TestCompletedCheckpoint checkpoint1 = CompletedCheckpointStoreTest.createCheckpoint(0, sharedStateRegistry);

		checkpointStore.addCheckpoint(checkpoint1);
		assertThat(checkpointStore.getAllCheckpoints(), Matchers.contains(checkpoint1));

		checkpointStore.shutdown(JobStatus.FINISHED);

		// verify that the checkpoint is discarded
		CompletedCheckpointStoreTest.verifyCheckpointDiscarded(checkpoint1);
	} finally {
		client.close();
	}
}
 
Example #29
Source Project: flink   Author: apache   File: BlobServerCorruptionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the GET operation fails when the downloaded file (from {@link BlobServer} or HA store)
 * is corrupt, i.e. its content's hash does not match the {@link BlobKey}'s hash.
 */
@Test
public void testGetFailsFromCorruptFile() throws IOException {

	final Configuration config = new Configuration();
	config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
	config.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());
	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, temporaryFolder.newFolder().getPath());

	BlobStoreService blobStoreService = null;

	try {
		blobStoreService = BlobUtils.createBlobStoreFromConfig(config);

		testGetFailsFromCorruptFile(config, blobStoreService, exception);
	} finally {
		if (blobStoreService != null) {
			blobStoreService.closeAndCleanupAllData();
		}
	}
}
 
Example #30
Source Project: flink   Author: apache   File: MesosServicesUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a {@link MesosServices} instance depending on the high availability settings.
 *
 * @param configuration containing the high availability settings
 * @param hostname the hostname to advertise to remote clients
 * @return a mesos services instance
 * @throws Exception if the mesos services instance could not be created
 */
public static MesosServices createMesosServices(Configuration configuration, String hostname) throws Exception {

	ActorSystem localActorSystem = AkkaUtils.createLocalActorSystem(configuration);

	MesosArtifactServer artifactServer = createArtifactServer(configuration, hostname);

	HighAvailabilityMode highAvailabilityMode = HighAvailabilityMode.fromConfig(configuration);

	switch (highAvailabilityMode) {
		case NONE:
			return new StandaloneMesosServices(localActorSystem, artifactServer);

		case ZOOKEEPER:
			final String zkMesosRootPath = configuration.getString(
				HighAvailabilityOptions.HA_ZOOKEEPER_MESOS_WORKERS_PATH);

			ZooKeeperUtilityFactory zooKeeperUtilityFactory = new ZooKeeperUtilityFactory(
				configuration,
				zkMesosRootPath);

			return new ZooKeeperMesosServices(localActorSystem, artifactServer, zooKeeperUtilityFactory);

		default:
			throw new Exception("High availability mode " + highAvailabilityMode + " is not supported.");
	}
}