org.apache.flink.runtime.testutils.CommonTestUtils Java Examples
The following examples show how to use
org.apache.flink.runtime.testutils.CommonTestUtils.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static void verifyStateRestore( JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex, List<KeyGroupRange> keyGroupPartitions) throws Exception { for (int i = 0; i < executionJobVertex.getParallelism(); i++) { JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore(); Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId()); TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot(); OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID)); ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend = generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false); assertTrue(CommonTestUtils.isSteamContentEqual( expectedOpStateBackend.get(0).openInputStream(), operatorState.getManagedOperatorState().iterator().next().openInputStream())); KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState( jobVertexID, keyGroupPartitions.get(i), false); compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState()); } }
Example #2
Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0 | 6 votes |
public static void verifyStateRestore( JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex, List<KeyGroupRange> keyGroupPartitions) throws Exception { for (int i = 0; i < executionJobVertex.getParallelism(); i++) { JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore(); Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId()); TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot(); OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID)); ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend = generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false); assertTrue(CommonTestUtils.isStreamContentEqual( expectedOpStateBackend.get(0).openInputStream(), operatorState.getManagedOperatorState().iterator().next().openInputStream())); KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState( jobVertexID, keyGroupPartitions.get(i), false); compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState()); } }
Example #3
Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0 | 6 votes |
public static void verifyStateRestore( JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex, List<KeyGroupRange> keyGroupPartitions) throws Exception { for (int i = 0; i < executionJobVertex.getParallelism(); i++) { JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore(); Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId()); TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot(); OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID)); ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend = generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false); assertTrue(CommonTestUtils.isStreamContentEqual( expectedOpStateBackend.get(0).openInputStream(), operatorState.getManagedOperatorState().iterator().next().openInputStream())); KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState( jobVertexID, keyGroupPartitions.get(i), false); compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState()); } }
Example #4
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRestarted( final RestClusterClient<ApplicationId> restClusterClient, final JobID jobId, final int expectedFullRestarts) throws Exception { CommonTestUtils.waitUntilCondition( () -> getJobFullRestarts(restClusterClient, jobId) >= expectedFullRestarts, Deadline.fromNow(TIMEOUT)); }
Example #5
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRunning(RestClusterClient<ApplicationId> restClusterClient, JobID jobId) throws Exception { CommonTestUtils.waitUntilCondition( () -> { final JobDetailsInfo jobDetails = restClusterClient.getJobDetails(jobId).get(); return jobDetails.getJobVertexInfos() .stream() .map(toExecutionState()) .allMatch(isRunning()); }, Deadline.fromNow(TIMEOUT)); }
Example #6
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private void killApplicationAndWait(final ApplicationId id) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); yarnClient.killApplication(id); CommonTestUtils.waitUntilCondition(() -> !yarnClient.getApplications(EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FINISHED)).isEmpty(), Deadline.fromNow(TIMEOUT)); }
Example #7
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); CommonTestUtils.waitUntilCondition(() -> { final ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); return applicationReport.getCurrentApplicationAttemptId().getAttemptId() >= attemptId; }, Deadline.fromNow(TIMEOUT)); }
Example #8
Source File: TaskExecutorITCase.java From flink with Apache License 2.0 | 5 votes |
private CompletableFuture<JobResult> submitJobAndWaitUntilRunning(JobGraph jobGraph) throws Exception { miniCluster.submitJob(jobGraph).get(); final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID()); assertThat(jobResultFuture.isDone(), is(false)); CommonTestUtils.waitUntilCondition( jobIsRunning(() -> miniCluster.getExecutionGraph(jobGraph.getJobID())), Deadline.fromNow(TESTING_TIMEOUT), 50L); return jobResultFuture; }
Example #9
Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
private void waitUntilAllExecutionsAreScheduled(final JobMasterGateway jobMasterGateway) throws Exception { final Duration duration = Duration.ofMillis(testingTimeout.toMilliseconds()); final Deadline deadline = Deadline.fromNow(duration); CommonTestUtils.waitUntilCondition( () -> getExecutions(jobMasterGateway).stream().allMatch(execution -> execution.getState() == ExecutionState.SCHEDULED), deadline); }
Example #10
Source File: TestProcessBuilder.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public TestProcessBuilder(String mainClass) throws IOException { File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties"); tempLogFile.deleteOnExit(); CommonTestUtils.printLog4jDebugConfig(tempLogFile); jvmArgs.add("-Dlog.level=DEBUG"); jvmArgs.add("-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath()); jvmArgs.add("-classpath"); jvmArgs.add(getCurrentClasspath()); this.mainClass = mainClass; }
Example #11
Source File: TaskManagerLoadingDynamicPropertiesITCase.java From flink with Apache License 2.0 | 5 votes |
@Test public void testLoadingDynamicPropertiesInBash() throws Exception { final Configuration clientConfiguration = new Configuration(); final File root = folder.getRoot(); final File homeDir = new File(root, "home"); assertTrue(homeDir.mkdir()); BootstrapTools.writeConfiguration(clientConfiguration, new File(homeDir, FLINK_CONF_FILENAME)); final Configuration jmUpdatedConfiguration = getJobManagerUpdatedConfiguration(); final File shellScriptFile = generateLaunchContainerScript( homeDir, BootstrapTools.getDynamicPropertiesAsString(clientConfiguration, jmUpdatedConfiguration)); Process process = new ProcessBuilder(shellScriptFile.getAbsolutePath()).start(); try { final StringWriter processOutput = new StringWriter(); new CommonTestUtils.PipeForwarder(process.getErrorStream(), processOutput); if (!process.waitFor(10, TimeUnit.SECONDS)) { throw new Exception("TestingTaskManagerRunner did not shutdown in time."); } assertEquals(processOutput.toString(), 0, process.exitValue()); } finally { process.destroy(); } }
Example #12
Source File: TestProcessBuilder.java From flink with Apache License 2.0 | 5 votes |
public TestProcessBuilder(String mainClass) throws IOException { File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties"); tempLogFile.deleteOnExit(); CommonTestUtils.printLog4jDebugConfig(tempLogFile); jvmArgs.add("-Dlog.level=DEBUG"); jvmArgs.add("-Dlog4j.configurationFile=file:" + tempLogFile.getAbsolutePath()); jvmArgs.add("-classpath"); jvmArgs.add(getCurrentClasspath()); this.mainClass = mainClass; }
Example #13
Source File: CliTableauResultViewTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testCancelBatchResult() throws Exception { ResultDescriptor resultDescriptor = new ResultDescriptor("", schema, true, true); TestingExecutor mockExecutor = new TestingExecutorBuilder() .setSnapshotResultSupplier(TypedResult::empty) .build(); CliTableauResultView view = new CliTableauResultView( terminal, mockExecutor, "session", resultDescriptor); // submit result display in another thread ExecutorService executorService = Executors.newSingleThreadExecutor(); Future<?> furture = executorService.submit(view::displayBatchResults); // wait until we trying to get batch result CommonTestUtils.waitUntilCondition( () -> mockExecutor.getNumSnapshotResultCalls() > 0, Deadline.now().plus(Duration.ofSeconds(5)), 50L); // send signal to cancel terminal.raise(Terminal.Signal.INT); furture.get(5, TimeUnit.SECONDS); Assert.assertEquals("Query terminated" + System.lineSeparator(), terminalOutput.toString()); // didn't have a chance to read page assertThat(mockExecutor.getNumRetrieveResultPageCalls(), is(0)); // tried to cancel query assertThat(mockExecutor.getNumCancelCalls(), is(1)); view.close(); }
Example #14
Source File: YARNSessionCapacitySchedulerITCase.java From flink with Apache License 2.0 | 5 votes |
private static void assertNumberOfSlotsPerTask( final String host, final int port, final int slotsNumber) throws Exception { try { CommonTestUtils.waitUntilCondition(() -> getNumberOfSlotsPerTaskManager(host, port) == slotsNumber, Deadline.fromNow(Duration.ofSeconds(30))); } catch (final TimeoutException e) { final int currentNumberOfSlots = getNumberOfSlotsPerTaskManager(host, port); fail(String.format("Expected slots per TM to be %d, was: %d", slotsNumber, currentNumberOfSlots)); } }
Example #15
Source File: YARNSessionCapacitySchedulerITCase.java From flink with Apache License 2.0 | 5 votes |
private static void assertNumberOfSlotsPerTask( final String host, final int port, final int slotsNumber) throws Exception { try { CommonTestUtils.waitUntilCondition(() -> getNumberOfSlotsPerTaskManager(host, port) == slotsNumber, Deadline.fromNow(Duration.ofSeconds(30))); } catch (final TimeoutException e) { final int currentNumberOfSlots = getNumberOfSlotsPerTaskManager(host, port); fail(String.format("Expected slots per TM to be %d, was: %d", slotsNumber, currentNumberOfSlots)); } }
Example #16
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRestarted( final RestClusterClient<ApplicationId> restClusterClient, final JobID jobId, final int expectedFullRestarts) throws Exception { CommonTestUtils.waitUntilCondition( () -> getJobFullRestarts(restClusterClient, jobId) >= expectedFullRestarts, Deadline.fromNow(TIMEOUT)); }
Example #17
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRunning(RestClusterClient<ApplicationId> restClusterClient, JobID jobId) throws Exception { CommonTestUtils.waitUntilCondition( () -> { final JobDetailsInfo jobDetails = restClusterClient.getJobDetails(jobId).get(); return jobDetails.getJobVertexInfos() .stream() .map(toExecutionState()) .allMatch(isRunning()); }, Deadline.fromNow(TIMEOUT)); }
Example #18
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private void killApplicationAndWait(final ApplicationId id) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); yarnClient.killApplication(id); CommonTestUtils.waitUntilCondition(() -> !yarnClient.getApplications(EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FINISHED)).isEmpty(), Deadline.fromNow(TIMEOUT)); }
Example #19
Source File: YARNHighAvailabilityITCase.java From flink with Apache License 2.0 | 5 votes |
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); CommonTestUtils.waitUntilCondition(() -> { final ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); return applicationReport.getCurrentApplicationAttemptId().getAttemptId() >= attemptId; }, Deadline.fromNow(TIMEOUT)); }
Example #20
Source File: TaskExecutorITCase.java From flink with Apache License 2.0 | 5 votes |
private CompletableFuture<JobResult> submitJobAndWaitUntilRunning(JobGraph jobGraph) throws Exception { miniCluster.submitJob(jobGraph).get(); final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID()); assertThat(jobResultFuture.isDone(), is(false)); CommonTestUtils.waitUntilCondition( jobIsRunning(() -> miniCluster.getExecutionGraph(jobGraph.getJobID())), Deadline.fromNow(TESTING_TIMEOUT), 50L); return jobResultFuture; }
Example #21
Source File: JobMasterTest.java From flink with Apache License 2.0 | 5 votes |
private void waitUntilAllExecutionsAreScheduled(final JobMasterGateway jobMasterGateway) throws Exception { final Duration duration = Duration.ofMillis(testingTimeout.toMilliseconds()); final Deadline deadline = Deadline.fromNow(duration); CommonTestUtils.waitUntilCondition( () -> getExecutions(jobMasterGateway).stream().allMatch(execution -> execution.getState() == ExecutionState.SCHEDULED), deadline); }
Example #22
Source File: TestProcessBuilder.java From flink with Apache License 2.0 | 5 votes |
public TestProcessBuilder(String mainClass) throws IOException { File tempLogFile = File.createTempFile(getClass().getSimpleName() + "-", "-log4j.properties"); tempLogFile.deleteOnExit(); CommonTestUtils.printLog4jDebugConfig(tempLogFile); jvmArgs.add("-Dlog.level=DEBUG"); jvmArgs.add("-Dlog4j.configuration=file:" + tempLogFile.getAbsolutePath()); jvmArgs.add("-classpath"); jvmArgs.add(getCurrentClasspath()); this.mainClass = mainClass; }
Example #23
Source File: YARNSessionCapacitySchedulerITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void assertNumberOfSlotsPerTask( final String host, final int port, final int slotsNumber) throws Exception { try { CommonTestUtils.waitUntilCondition(() -> getNumberOfSlotsPerTaskManager(host, port) == slotsNumber, Deadline.fromNow(Duration.ofSeconds(30))); } catch (final TimeoutException e) { final int currentNumberOfSlots = getNumberOfSlotsPerTaskManager(host, port); fail(String.format("Expected slots per TM to be %d, was: %d", slotsNumber, currentNumberOfSlots)); } }
Example #24
Source File: TaskExecutorITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private CompletableFuture<JobResult> submitJobAndWaitUntilRunning(JobGraph jobGraph) throws Exception { miniCluster.submitJob(jobGraph).get(); final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID()); assertThat(jobResultFuture.isDone(), is(false)); CommonTestUtils.waitUntilCondition( jobIsRunning(() -> miniCluster.getExecutionGraph(jobGraph.getJobID())), Deadline.fromNow(TESTING_TIMEOUT), 50L); return jobResultFuture; }
Example #25
Source File: YARNHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRestarted( final RestClusterClient<ApplicationId> restClusterClient, final JobID jobId, final int expectedFullRestarts) throws Exception { CommonTestUtils.waitUntilCondition( () -> getJobFullRestarts(restClusterClient, jobId) >= expectedFullRestarts, Deadline.fromNow(TIMEOUT)); }
Example #26
Source File: YARNHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private static void waitUntilJobIsRunning(RestClusterClient<ApplicationId> restClusterClient, JobID jobId) throws Exception { CommonTestUtils.waitUntilCondition( () -> { final JobDetailsInfo jobDetails = restClusterClient.getJobDetails(jobId).get(); return jobDetails.getJobVertexInfos() .stream() .map(toExecutionState()) .allMatch(isRunning()); }, Deadline.fromNow(TIMEOUT)); }
Example #27
Source File: BackPressureStatsTrackerImplITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void waitUntilBackPressureStatsAvailable() { try { CommonTestUtils.waitUntilCondition( () -> { final Optional<OperatorBackPressureStats> stats = getBackPressureStats(); return stats.isPresent(); }, Deadline.fromNow(TIMEOUT)); } catch (Exception e) { throw new RuntimeException(e); } }
Example #28
Source File: YARNHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void killApplicationAndWait(final ApplicationId id) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); yarnClient.killApplication(id); CommonTestUtils.waitUntilCondition(() -> !yarnClient.getApplications(EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FINISHED)).isEmpty(), Deadline.fromNow(TIMEOUT)); }
Example #29
Source File: YARNHighAvailabilityITCase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception { final YarnClient yarnClient = getYarnClient(); checkState(yarnClient != null, "yarnClient must be initialized"); CommonTestUtils.waitUntilCondition(() -> { final ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId); return applicationReport.getCurrentApplicationAttemptId().getAttemptId() >= attemptId; }, Deadline.fromNow(TIMEOUT)); }
Example #30
Source File: LeaderChangeClusterComponentsTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private void waitUntilTaskExecutorsHaveConnected(int numTaskExecutors, Deadline deadline) throws Exception { CommonTestUtils.waitUntilCondition( () -> miniCluster.requestClusterOverview().get().getNumTaskManagersConnected() == numTaskExecutors, deadline, 10L); }