Java Code Examples for org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration

The following examples show how to use org.apache.flink.runtime.testutils.MiniClusterResourceConfiguration. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: RollingSinkITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {

	LOG.info("In RollingSinkITCase: Starting MiniDFSCluster ");

	dataDir = tempFolder.newFolder();

	conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, dataDir.getAbsolutePath());
	MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
	hdfsCluster = builder.build();

	dfs = hdfsCluster.getFileSystem();

	hdfsURI = "hdfs://"
			+ NetUtils.hostAndPortToUrlString(hdfsCluster.getURI().getHost(), hdfsCluster.getNameNodePort())
			+ "/";

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(4)
			.build());

	miniClusterResource.before();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 3
Source Project: Flink-CEPplus   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: NettyEpollITCase.java    License: Apache License 2.0 6 votes vote down vote up
private MiniClusterWithClientResource trySetUpCluster() throws Exception {
	try {
		Configuration config = new Configuration();
		config.setString(TRANSPORT_TYPE, "epoll");
		MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(NUM_TASK_MANAGERS)
				.setNumberSlotsPerTaskManager(1)
				.build());
		cluster.before();
		return cluster;
	}
	catch (UnsatisfiedLinkError ex) {
		// If we failed to init netty because we are not on Linux platform, abort the test.
		if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) {
			throw new AssumptionViolatedException("This test is only supported on linux");
		}
		throw ex;
	}
}
 
Example 5
Source Project: Flink-CEPplus   Source File: HistoryServerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	jmDirectory = TMP.newFolder("jm_" + versionLessThan14);
	hsDirectory = TMP.newFolder("hs_" + versionLessThan14);

	Configuration clusterConfig = new Configuration();
	clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(clusterConfig)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1)
			.build());
	cluster.before();
}
 
Example 6
Source Project: Flink-CEPplus   Source File: BlobsCleanupITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(ConfigConstants.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 
Example 7
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 8
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 9
Source Project: flink   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "80m");

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 10
Source Project: flink   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 11
Source Project: flink   Source File: StreamFaultToleranceTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	switch (failoverStrategy) {
		case RestartPipelinedRegionStrategy:
			configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region");
			break;
		case RestartAllStrategy:
			configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "full");
	}

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(NUM_TASK_MANAGERS)
			.setNumberSlotsPerTaskManager(NUM_TASK_SLOTS)
			.build());
	cluster.before();
}
 
Example 12
Source Project: flink   Source File: RegionFailoverITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region");
	// global failover times: 3, region failover times: NUM_OF_RESTARTS
	configuration.setInteger(FailingRestartStrategy.NUM_FAILURES_CONFIG_OPTION, 3);
	configuration.setString(ConfigConstants.RESTART_STRATEGY, FailingRestartStrategy.class.getName());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2).build());
	cluster.before();
	jobFailedCnt.set(0);
	numCompletedCheckpoints.set(0);
}
 
Example 13
Source Project: flink   Source File: NettyEpollITCase.java    License: Apache License 2.0 6 votes vote down vote up
private MiniClusterWithClientResource trySetUpCluster() throws Exception {
	try {
		Configuration config = new Configuration();
		config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll");
		MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(NUM_TASK_MANAGERS)
				.setNumberSlotsPerTaskManager(1)
				.build());
		cluster.before();
		return cluster;
	}
	catch (UnsatisfiedLinkError ex) {
		// If we failed to init netty because we are not on Linux platform, abort the test.
		if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) {
			throw new AssumptionViolatedException("This test is only supported on linux");
		}
		throw ex;
	}
}
 
Example 14
Source Project: flink   Source File: HistoryServerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	jmDirectory = TMP.newFolder("jm_" + versionLessThan14);
	hsDirectory = TMP.newFolder("hs_" + versionLessThan14);

	Configuration clusterConfig = new Configuration();
	clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(clusterConfig)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1)
			.build());
	cluster.before();
}
 
Example 15
Source Project: flink   Source File: BlobsCleanupITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(ConfigConstants.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 
Example 16
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 17
Source Project: flink   Source File: HAQueryableStateFsBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 18
Source Project: flink   Source File: ClassLoaderITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {

	Configuration config = new Configuration();

	// we need to use the "filesystem" state backend to ensure FLINK-2543 is not happening again.
	config.setString(CheckpointingOptions.STATE_BACKEND, "filesystem");
	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// Savepoint path
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY,
			FOLDER.newFolder().getAbsoluteFile().toURI().toString());

	// required as we otherwise run out of memory
	config.set(TaskManagerOptions.MANAGED_MEMORY_SIZE, MemorySize.parse("80m"));

	miniClusterResource = new MiniClusterResource(
		new MiniClusterResourceConfiguration.Builder()
			.setNumberTaskManagers(2)
			.setNumberSlotsPerTaskManager(2)
			.setConfiguration(config)
			.build());

	miniClusterResource.before();
}
 
Example 19
Source Project: flink   Source File: RescalingITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	// detect parameter change
	if (currentBackend != backend) {
		shutDownExistingCluster();

		currentBackend = backend;

		Configuration config = new Configuration();

		final File checkpointDir = temporaryFolder.newFolder();
		final File savepointDir = temporaryFolder.newFolder();

		config.setString(CheckpointingOptions.STATE_BACKEND, currentBackend);
		config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
		config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

		cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlots)
				.build());
		cluster.before();
	}
}
 
Example 20
Source Project: flink   Source File: NotifyCheckpointAbortedITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 
Example 21
Source Project: flink   Source File: StreamFaultToleranceTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	switch (failoverStrategy) {
		case RestartPipelinedRegionFailoverStrategy:
			configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "region");
			break;
		case RestartAllFailoverStrategy:
			configuration.setString(JobManagerOptions.EXECUTION_FAILOVER_STRATEGY, "full");
	}

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(NUM_TASK_MANAGERS)
			.setNumberSlotsPerTaskManager(NUM_TASK_SLOTS)
			.build());
	cluster.before();
}
 
Example 22
Source Project: flink   Source File: NettyEpollITCase.java    License: Apache License 2.0 6 votes vote down vote up
private MiniClusterWithClientResource trySetUpCluster() throws Exception {
	try {
		Configuration config = new Configuration();
		config.setString(NettyShuffleEnvironmentOptions.TRANSPORT_TYPE, "epoll");
		MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setConfiguration(config)
				.setNumberTaskManagers(NUM_TASK_MANAGERS)
				.setNumberSlotsPerTaskManager(1)
				.build());
		cluster.before();
		return cluster;
	}
	catch (UnsatisfiedLinkError ex) {
		// If we failed to init netty because we are not on Linux platform, abort the test.
		if (findThrowableWithMessage(ex, "Only supported on Linux").isPresent()) {
			throw new AssumptionViolatedException("This test is only supported on linux");
		}
		throw ex;
	}
}
 
Example 23
Source Project: flink   Source File: HistoryServerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
	jmDirectory = tmpFolder.newFolder("jm_" + versionLessThan14);
	hsDirectory = tmpFolder.newFolder("hs_" + versionLessThan14);

	Configuration clusterConfig = new Configuration();
	clusterConfig.setString(JobManagerOptions.ARCHIVE_DIR, jmDirectory.toURI().toString());

	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(clusterConfig)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1)
			.build());
	cluster.before();
}
 
Example 24
Source Project: flink   Source File: BlobsCleanupITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	blobBaseDir = TEMPORARY_FOLDER.newFolder();

	Configuration cfg = new Configuration();
	cfg.setString(BlobServerOptions.STORAGE_DIRECTORY, blobBaseDir.getAbsolutePath());
	cfg.setString(RestartStrategyOptions.RESTART_STRATEGY, "fixeddelay");
	cfg.setInteger(RestartStrategyOptions.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	// BLOBs are deleted from BlobCache between 1s and 2s after last reference
	// -> the BlobCache may still have the BLOB or not (let's test both cases randomly)
	cfg.setLong(BlobServerOptions.CLEANUP_INTERVAL, 1L);

	configuration = new UnmodifiableConfiguration(cfg);

	miniClusterResource = new MiniClusterResource(new MiniClusterResourceConfiguration.Builder()
		.setNumberSlotsPerTaskManager(2)
		.setNumberTaskManagers(1)
		.setConfiguration(configuration)
		.build());

	miniClusterResource.before();
}
 
Example 25
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 26
Source Project: flink   Source File: HAQueryableStateFsBackendITCase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfig())
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();

	client = new QueryableStateClient("localhost", QS_PROXY_PORT_RANGE_START);

	clusterClient = miniClusterResource.getClusterClient();
}
 
Example 27
@BeforeClass
public static void setup() throws Exception {
	zkServer = new TestingServer();

	Configuration config = new Configuration();
	config.setInteger(ConfigConstants.LOCAL_NUMBER_JOB_MANAGER, NUM_JMS);
	config.setInteger(ConfigConstants.LOCAL_NUMBER_TASK_MANAGER, NUM_TMS);
	config.setInteger(TaskManagerOptions.NUM_TASK_SLOTS, NUM_SLOTS_PER_TM);

	haStorageDir = TEMPORARY_FOLDER.newFolder();

	config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haStorageDir.toString());
	config.setString(HighAvailabilityOptions.HA_CLUSTER_ID, UUID.randomUUID().toString());
	config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zkServer.getConnectString());
	config.setString(HighAvailabilityOptions.HA_MODE, "zookeeper");

	config.setString(
		ConfigConstants.METRICS_REPORTER_PREFIX + "restarts." + ConfigConstants.METRICS_REPORTER_CLASS_SUFFIX,
		RestartReporter.class.getName());

	// we have to manage this manually because we have to create the ZooKeeper server
	// ahead of this
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(NUM_TMS)
			.setNumberSlotsPerTaskManager(NUM_SLOTS_PER_TM)
			.build());

	miniClusterResource.before();
}
 
Example 28
Source Project: Flink-CEPplus   Source File: SavepointMigrationTestBase.java    License: Apache License 2.0 5 votes vote down vote up
protected SavepointMigrationTestBase() throws Exception {
	miniClusterResource = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(getConfiguration())
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(DEFAULT_PARALLELISM)
			.build());
}
 
Example 29
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerSavepointForNonExistingJob() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobID jobID = new JobID();

	try {
		client.triggerSavepoint(jobID, null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, jobID.toString()).isPresent());
	} finally {
		cluster.after();
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		client.setDetached(true);
		client.submitJob(graph, SavepointITCase.class.getClassLoader());

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}