Java Code Examples for org.apache.flink.test.util.MiniClusterWithClientResource#after()

The following examples show how to use org.apache.flink.test.util.MiniClusterWithClientResource#after() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		client.setDetached(true);
		client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());

		StatefulCounter.getProgressLatch().await();

		return client.cancelWithSavepoint(jobId, null);
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}

Example 2

Source File: SavepointITCase.java From flink with Apache License 2.0

6 votes

private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		ClientUtils.submitJob(client, jobGraph);

		StatefulCounter.getProgressLatch().await();

		return client.cancelWithSavepoint(jobId, null).get();
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}

Example 3

Source File: NettyEpollITCase.java From flink with Apache License 2.0

6 votes

@Test
public void testNettyEpoll() throws Exception {
	MiniClusterWithClientResource cluster = trySetUpCluster();
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(NUM_TASK_MANAGERS);
		env.getConfig().disableSysoutLogging();

		DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42);
		input.keyBy(new KeySelector<Integer, Integer>() {
				@Override
				public Integer getKey(Integer value) throws Exception {
					return value;
				}
			})
			.sum(0)
			.print();

		env.execute();
	}
	finally {
		cluster.after();
	}
}

Example 4

Source File: NettyEpollITCase.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testNettyEpoll() throws Exception {
	MiniClusterWithClientResource cluster = trySetUpCluster();
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(NUM_TASK_MANAGERS);
		env.getConfig().disableSysoutLogging();

		DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42);
		input.keyBy(new KeySelector<Integer, Integer>() {
				@Override
				public Integer getKey(Integer value) throws Exception {
					return value;
				}
			})
			.sum(0)
			.print();

		env.execute();
	}
	finally {
		cluster.after();
	}
}

Example 5

Source File: SavepointITCase.java From flink with Apache License 2.0

6 votes

private String submitJobAndTakeSavepoint(MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		client.setDetached(true);
		client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());

		StatefulCounter.getProgressLatch().await();

		return client.cancelWithSavepoint(jobId, null);
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}

Example 6

Source File: NettyEpollITCase.java From flink with Apache License 2.0

6 votes

@Test
public void testNettyEpoll() throws Exception {
	MiniClusterWithClientResource cluster = trySetUpCluster();
	try {
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(NUM_TASK_MANAGERS);

		DataStream<Integer> input = env.fromElements(1, 2, 3, 4, 1, 2, 3, 42);
		input.keyBy(new KeySelector<Integer, Integer>() {
				@Override
				public Integer getKey(Integer value) throws Exception {
					return value;
				}
			})
			.sum(0)
			.print();

		env.execute();
	}
	finally {
		cluster.after();
	}
}

Example 7

Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

private void restoreJobAndVerifyState(String savepointPath, MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		client.setDetached(true);
		client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());

		// Await state is restored
		StatefulCounter.getRestoreLatch().await();

		// Await some progress after restore
		StatefulCounter.getProgressLatch().await();

		client.cancel(jobId);

		FutureUtils.retrySuccessfulWithDelay(
			() -> client.getJobStatus(jobId),
			Time.milliseconds(50),
			Deadline.now().plus(Duration.ofSeconds(30)),
			status -> status == JobStatus.CANCELED,
			TestingUtils.defaultScheduledExecutor()
		);

		client.disposeSavepoint(savepointPath)
			.get();

		assertFalse("Savepoint not properly cleaned up.", new File(savepointPath).exists());
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}

Example 8

Source File: SavepointITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		ClientUtils.submitJob(client, graph);

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}

Example 9

Source File: SavepointITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testTriggerSavepointForNonExistingJob() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobID jobID = new JobID();

	try {
		client.triggerSavepoint(jobID, null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, jobID.toString()).isPresent());
	} finally {
		cluster.after();
	}
}

Example 10

Source File: SavepointITCase.java From flink with Apache License 2.0

5 votes

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		client.setDetached(true);
		client.submitJob(graph, SavepointITCase.class.getClassLoader());

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}

Example 11

Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testTriggerSavepointWithCheckpointingDisabled() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobVertex vertex = new JobVertex("Blocking vertex");
	vertex.setInvokableClass(BlockingNoOpInvokable.class);
	vertex.setParallelism(1);

	final JobGraph graph = new JobGraph(vertex);

	try {
		client.setDetached(true);
		client.submitJob(graph, SavepointITCase.class.getClassLoader());

		client.triggerSavepoint(graph.getJobID(), null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, IllegalStateException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, graph.getJobID().toString()).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, "is not a streaming job").isPresent());
	} finally {
		cluster.after();
	}
}

Example 12

Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testTriggerSavepointForNonExistingJob() throws Exception {
	// Config
	final int numTaskManagers = 1;
	final int numSlotsPerTaskManager = 1;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	final ClusterClient<?> client = cluster.getClusterClient();

	final JobID jobID = new JobID();

	try {
		client.triggerSavepoint(jobID, null).get();

		fail();
	} catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, FlinkJobNotFoundException.class).isPresent());
		assertTrue(ExceptionUtils.findThrowableWithMessage(e, jobID.toString()).isPresent());
	} finally {
		cluster.after();
	}
}

Example 13

Source File: SavepointITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testSubmitWithUnknownSavepointPath() throws Exception {
	// Config
	int numTaskManagers = 1;
	int numSlotsPerTaskManager = 1;
	int parallelism = numTaskManagers * numSlotsPerTaskManager;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {

		// High value to ensure timeouts if restarted.
		int numberOfRetries = 1000;
		// Submit the job
		// Long delay to ensure that the test times out if the job
		// manager tries to restart the job.
		final JobGraph jobGraph = createJobGraph(parallelism, numberOfRetries, 3600000);

		// Set non-existing savepoint path
		jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("unknown path"));
		assertEquals("unknown path", jobGraph.getSavepointRestoreSettings().getRestorePath());

		LOG.info("Submitting job " + jobGraph.getJobID() + " in detached mode.");

		try {
			ClientUtils.submitJobAndWaitForResult(client, jobGraph, SavepointITCase.class.getClassLoader());
		} catch (Exception e) {
			Optional<JobExecutionException> expectedJobExecutionException = ExceptionUtils.findThrowable(e, JobExecutionException.class);
			Optional<FileNotFoundException> expectedFileNotFoundException = ExceptionUtils.findThrowable(e, FileNotFoundException.class);
			if (!(expectedJobExecutionException.isPresent() && expectedFileNotFoundException.isPresent())) {
				throw e;
			}
		}
	} finally {
		cluster.after();
	}
}

Example 14

Source File: NetworkStackThroughputITCase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testThroughput() throws Exception {
	Object[][] configParams = new Object[][]{
			new Object[]{1, false, false, false, 4, 2},
			new Object[]{1, true, false, false, 4, 2},
			new Object[]{1, true, true, false, 4, 2},
			new Object[]{1, true, false, true, 4, 2},
			new Object[]{2, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 8, 4},
	};

	for (Object[] p : configParams) {
		final int dataVolumeGb = (Integer) p[0];
		final boolean useForwarder = (Boolean) p[1];
		final boolean isSlowSender = (Boolean) p[2];
		final boolean isSlowReceiver = (Boolean) p[3];
		final int parallelism = (Integer) p[4];
		final int numSlotsPerTaskManager = (Integer) p[5];

		if (parallelism % numSlotsPerTaskManager != 0) {
			throw new RuntimeException("The test case defines a parallelism that is not a multiple of the slots per task manager.");
		}

		final int numTaskManagers = parallelism / numSlotsPerTaskManager;

		final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
				.build());
		cluster.before();

		try {
			System.out.println(String.format("Running test with parameters: dataVolumeGB=%s, useForwarder=%s, isSlowSender=%s, isSlowReceiver=%s, parallelism=%s, numSlotsPerTM=%s",
				dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism, numSlotsPerTaskManager));
			testProgram(
				cluster,
				dataVolumeGb,
				useForwarder,
				isSlowSender,
				isSlowReceiver,
				parallelism);
		} finally {
			cluster.after();
		}
	}
}

Example 15

Source File: SavepointITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testSubmitWithUnknownSavepointPath() throws Exception {
	// Config
	int numTaskManagers = 1;
	int numSlotsPerTaskManager = 1;
	int parallelism = numTaskManagers * numSlotsPerTaskManager;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {

		// High value to ensure timeouts if restarted.
		int numberOfRetries = 1000;
		// Submit the job
		// Long delay to ensure that the test times out if the job
		// manager tries to restart the job.
		final JobGraph jobGraph = createJobGraph(parallelism, numberOfRetries, 3600000);

		// Set non-existing savepoint path
		jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("unknown path"));
		assertEquals("unknown path", jobGraph.getSavepointRestoreSettings().getRestorePath());

		LOG.info("Submitting job " + jobGraph.getJobID() + " in detached mode.");

		try {
			client.setDetached(false);
			client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());
		} catch (Exception e) {
			Optional<JobExecutionException> expectedJobExecutionException = ExceptionUtils.findThrowable(e, JobExecutionException.class);
			Optional<FileNotFoundException> expectedFileNotFoundException = ExceptionUtils.findThrowable(e, FileNotFoundException.class);
			if (!(expectedJobExecutionException.isPresent() && expectedFileNotFoundException.isPresent())) {
				throw e;
			}
		}
	} finally {
		cluster.after();
	}
}

Example 16

Source File: NetworkStackThroughputITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testThroughput() throws Exception {
	Object[][] configParams = new Object[][]{
			new Object[]{1, false, false, false, 4, 2},
			new Object[]{1, true, false, false, 4, 2},
			new Object[]{1, true, true, false, 4, 2},
			new Object[]{1, true, false, true, 4, 2},
			new Object[]{2, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 8, 4},
	};

	for (Object[] p : configParams) {
		final int dataVolumeGb = (Integer) p[0];
		final boolean useForwarder = (Boolean) p[1];
		final boolean isSlowSender = (Boolean) p[2];
		final boolean isSlowReceiver = (Boolean) p[3];
		final int parallelism = (Integer) p[4];
		final int numSlotsPerTaskManager = (Integer) p[5];

		if (parallelism % numSlotsPerTaskManager != 0) {
			throw new RuntimeException("The test case defines a parallelism that is not a multiple of the slots per task manager.");
		}

		final int numTaskManagers = parallelism / numSlotsPerTaskManager;

		final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
				.build());
		cluster.before();

		try {
			System.out.println(String.format("Running test with parameters: dataVolumeGB=%s, useForwarder=%s, isSlowSender=%s, isSlowReceiver=%s, parallelism=%s, numSlotsPerTM=%s",
				dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism, numSlotsPerTaskManager));
			testProgram(
				cluster,
				dataVolumeGb,
				useForwarder,
				isSlowSender,
				isSlowReceiver,
				parallelism);
		} finally {
			cluster.after();
		}
	}
}

Example 17

Source File: NetworkStackThroughputITCase.java From flink with Apache License 2.0

4 votes

@Test
public void testThroughput() throws Exception {
	Object[][] configParams = new Object[][]{
			new Object[]{1, false, false, false, 4, 2},
			new Object[]{1, true, false, false, 4, 2},
			new Object[]{1, true, true, false, 4, 2},
			new Object[]{1, true, false, true, 4, 2},
			new Object[]{2, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 4, 2},
			new Object[]{4, true, false, false, 8, 4},
	};

	for (Object[] p : configParams) {
		final int dataVolumeGb = (Integer) p[0];
		final boolean useForwarder = (Boolean) p[1];
		final boolean isSlowSender = (Boolean) p[2];
		final boolean isSlowReceiver = (Boolean) p[3];
		final int parallelism = (Integer) p[4];
		final int numSlotsPerTaskManager = (Integer) p[5];

		if (parallelism % numSlotsPerTaskManager != 0) {
			throw new RuntimeException("The test case defines a parallelism that is not a multiple of the slots per task manager.");
		}

		final int numTaskManagers = parallelism / numSlotsPerTaskManager;

		final MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
			new MiniClusterResourceConfiguration.Builder()
				.setNumberTaskManagers(numTaskManagers)
				.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
				.build());
		cluster.before();

		try {
			System.out.println(String.format("Running test with parameters: dataVolumeGB=%s, useForwarder=%s, isSlowSender=%s, isSlowReceiver=%s, parallelism=%s, numSlotsPerTM=%s",
				dataVolumeGb, useForwarder, isSlowSender, isSlowReceiver, parallelism, numSlotsPerTaskManager));
			testProgram(
				cluster,
				dataVolumeGb,
				useForwarder,
				isSlowSender,
				isSlowReceiver,
				parallelism);
		} finally {
			cluster.after();
		}
	}
}

Example 18

Source File: ResumeCheckpointManuallyITCase.java From flink with Apache License 2.0

4 votes

private void testExternalizedCheckpoints(
	File checkpointDir,
	String zooKeeperQuorum,
	StateBackend backend,
	boolean localRecovery) throws Exception {

	final Configuration config = new Configuration();

	final File savepointDir = temporaryFolder.newFolder();

	config.setString(CheckpointingOptions.CHECKPOINTS_DIRECTORY, checkpointDir.toURI().toString());
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());
	config.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, localRecovery);

	// ZooKeeper recovery mode?
	if (zooKeeperQuorum != null) {
		final File haDir = temporaryFolder.newFolder();
		config.setString(HighAvailabilityOptions.HA_MODE, "ZOOKEEPER");
		config.setString(HighAvailabilityOptions.HA_ZOOKEEPER_QUORUM, zooKeeperQuorum);
		config.setString(HighAvailabilityOptions.HA_STORAGE_PATH, haDir.toURI().toString());
	}

	MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(NUM_TASK_MANAGERS)
			.setNumberSlotsPerTaskManager(SLOTS_PER_TASK_MANAGER)
			.build());

	cluster.before();

	ClusterClient<?> client = cluster.getClusterClient();

	try {
		// main test sequence:  start job -> eCP -> restore job -> eCP -> restore job
		String firstExternalCheckpoint = runJobAndGetExternalizedCheckpoint(backend, checkpointDir, null, client);
		assertNotNull(firstExternalCheckpoint);

		String secondExternalCheckpoint = runJobAndGetExternalizedCheckpoint(backend, checkpointDir, firstExternalCheckpoint, client);
		assertNotNull(secondExternalCheckpoint);

		String thirdExternalCheckpoint = runJobAndGetExternalizedCheckpoint(backend, checkpointDir, secondExternalCheckpoint, client);
		assertNotNull(thirdExternalCheckpoint);
	} finally {
		cluster.after();
	}
}

Example 19

Source File: SavepointITCase.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testSubmitWithUnknownSavepointPath() throws Exception {
	// Config
	int numTaskManagers = 1;
	int numSlotsPerTaskManager = 1;
	int parallelism = numTaskManagers * numSlotsPerTaskManager;

	final Configuration config = new Configuration();
	config.setString(CheckpointingOptions.SAVEPOINT_DIRECTORY, savepointDir.toURI().toString());

	MiniClusterWithClientResource cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(config)
			.setNumberTaskManagers(numTaskManagers)
			.setNumberSlotsPerTaskManager(numSlotsPerTaskManager)
			.build());
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {

		// High value to ensure timeouts if restarted.
		int numberOfRetries = 1000;
		// Submit the job
		// Long delay to ensure that the test times out if the job
		// manager tries to restart the job.
		final JobGraph jobGraph = createJobGraph(parallelism, numberOfRetries, 3600000);

		// Set non-existing savepoint path
		jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath("unknown path"));
		assertEquals("unknown path", jobGraph.getSavepointRestoreSettings().getRestorePath());

		LOG.info("Submitting job " + jobGraph.getJobID() + " in detached mode.");

		try {
			client.setDetached(false);
			client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());
		} catch (Exception e) {
			Optional<JobExecutionException> expectedJobExecutionException = ExceptionUtils.findThrowable(e, JobExecutionException.class);
			Optional<FileNotFoundException> expectedFileNotFoundException = ExceptionUtils.findThrowable(e, FileNotFoundException.class);
			if (!(expectedJobExecutionException.isPresent() && expectedFileNotFoundException.isPresent())) {
				throw e;
			}
		}
	} finally {
		cluster.after();
	}
}

Example 20

Source File: SavepointITCase.java From flink with Apache License 2.0

4 votes

private void restoreJobAndVerifyState(
	String savepointPath,
	MiniClusterResourceFactory clusterFactory,
	int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, false));
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		ClientUtils.submitJob(client, jobGraph);

		// Await state is restored
		StatefulCounter.getRestoreLatch().await();

		// Await some progress after restore
		StatefulCounter.getProgressLatch().await();

		client.cancel(jobId).get();

		FutureUtils.retrySuccessfulWithDelay(
			() -> client.getJobStatus(jobId),
			Time.milliseconds(50),
			Deadline.now().plus(Duration.ofSeconds(30)),
			status -> status == JobStatus.CANCELED,
			TestingUtils.defaultScheduledExecutor()
		);

		client.disposeSavepoint(savepointPath)
			.get();

		assertFalse("Savepoint not properly cleaned up.", new File(savepointPath).exists());
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}