Java Code Examples for org.apache.flink.api.common.time.Deadline

The following examples show how to use org.apache.flink.api.common.time.Deadline. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Called when resuming execution after a failure and waits until the lease
 * of the file we are resuming is free.
 *
 * <p>The lease of the file we are resuming writing/committing to may still
 * belong to the process that failed previously and whose state we are
 * recovering.
 *
 * @param path The path to the file we want to resume writing to.
 */
private static boolean waitUntilLeaseIsRevoked(final FileSystem fs, final Path path) throws IOException {
	Preconditions.checkState(fs instanceof DistributedFileSystem);

	final DistributedFileSystem dfs = (DistributedFileSystem) fs;
	dfs.recoverLease(path);

	final Deadline deadline = Deadline.now().plus(Duration.ofMillis(LEASE_TIMEOUT));

	final StopWatch sw = new StopWatch();
	sw.start();

	boolean isClosed = dfs.isFileClosed(path);
	while (!isClosed && deadline.hasTimeLeft()) {
		try {
			Thread.sleep(500L);
		} catch (InterruptedException e1) {
			throw new IOException("Recovering the lease failed: ", e1);
		}
		isClosed = dfs.isFileClosed(path);
	}
	return isClosed;
}
 
Example 2
Source Project: flink   Source File: YarnTestBase.java    License: Apache License 2.0 6 votes vote down vote up
protected void waitApplicationFinishedElseKillIt(
	ApplicationId applicationId,
	Duration timeout,
	YarnClusterDescriptor yarnClusterDescriptor,
	int sleepIntervalInMS) throws Exception {
	Deadline deadline = Deadline.now().plus(timeout);
	YarnApplicationState state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();

	while (state != YarnApplicationState.FINISHED) {
		if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) {
			Assert.fail("Application became FAILED or KILLED while expecting FINISHED");
		}

		if (deadline.isOverdue()) {
			yarnClusterDescriptor.killCluster(applicationId);
			Assert.fail("Application didn't finish before timeout");
		}

		sleep(sleepIntervalInMS);
		state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();
	}
}
 
Example 3
Source Project: flink   Source File: AutoClosableProcess.java    License: Apache License 2.0 6 votes vote down vote up
public void runBlockingWithRetry(final int maxRetries, final Duration attemptTimeout, final Duration globalTimeout) throws IOException {
	int retries = 0;
	final Deadline globalDeadline = Deadline.fromNow(globalTimeout);

	while (true) {
		try {
			runBlocking(attemptTimeout);
			break;
		} catch (Exception e) {
			if (++retries > maxRetries || !globalDeadline.hasTimeLeft()) {
				String errMsg = String.format(
					"Process (%s) exceeded timeout (%s) or number of retries (%s).",
					Arrays.toString(commands), globalTimeout.toMillis(), maxRetries);
				throw new IOException(errMsg, e);
			}
		}
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: SystemProcessingTimeService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean shutdownServiceUninterruptible(long timeoutMs) {

	final Deadline deadline = Deadline.fromNow(Duration.ofMillis(timeoutMs));

	boolean shutdownComplete = false;
	boolean receivedInterrupt = false;

	do {
		try {
			// wait for a reasonable time for all pending timer threads to finish
			shutdownComplete = shutdownAndAwaitPending(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		} catch (InterruptedException iex) {
			receivedInterrupt = true;
			LOG.trace("Intercepted attempt to interrupt timer service shutdown.", iex);
		}
	} while (deadline.hasTimeLeft() && !shutdownComplete);

	if (receivedInterrupt) {
		Thread.currentThread().interrupt();
	}

	return shutdownComplete;
}
 
Example 5
Source Project: flink   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Waits until all executions fulfill the given predicate.
 *
 * @param executionGraph for which to check the executions
 * @param executionPredicate predicate which is to be fulfilled
 * @param maxWaitMillis timeout for the wait operation
 * @throws TimeoutException if the executions did not reach the target state in time
 */
public static void waitForAllExecutionsPredicate(
		ExecutionGraph executionGraph,
		Predicate<AccessExecution> executionPredicate,
		long maxWaitMillis) throws TimeoutException {
	final Predicate<AccessExecutionGraph> allExecutionsPredicate = allExecutionsPredicate(executionPredicate);
	final Deadline deadline = Deadline.fromNow(Duration.ofMillis(maxWaitMillis));
	boolean predicateResult;

	do {
		predicateResult = allExecutionsPredicate.test(executionGraph);

		if (!predicateResult) {
			try {
				Thread.sleep(2L);
			} catch (InterruptedException ignored) {
				Thread.currentThread().interrupt();
			}
		}
	} while (!predicateResult && deadline.hasTimeLeft());

	if (!predicateResult) {
		throw new TimeoutException("Not all executions fulfilled the predicate in time.");
	}
}
 
Example 6
Source Project: flink   Source File: LeaderChangeClusterComponentsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskExecutorsReconnectToClusterWithLeadershipChange() throws Exception {
	final Deadline deadline = Deadline.fromNow(TESTING_TIMEOUT);
	waitUntilTaskExecutorsHaveConnected(NUM_TMS, deadline);
	highAvailabilityServices.revokeResourceManagerLeadership().get();
	highAvailabilityServices.grantResourceManagerLeadership();

	// wait for the ResourceManager to confirm the leadership
	assertThat(
		LeaderRetrievalUtils.retrieveLeaderConnectionInfo(
			highAvailabilityServices.getResourceManagerLeaderRetriever(),
			TESTING_TIMEOUT).getLeaderSessionId(),
		is(notNullValue()));

	waitUntilTaskExecutorsHaveConnected(NUM_TMS, deadline);
}
 
Example 7
Source Project: flink   Source File: AbstractOperatorRestoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private void restoreJob(ClusterClient<?> clusterClient, Deadline deadline, String savepointPath) throws Exception {
	JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE);
	jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState));

	assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID());

	ClientUtils.submitJob(clusterClient, jobToRestore);

	CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay(
		() -> clusterClient.getJobStatus(jobToRestore.getJobID()),
		Time.milliseconds(50),
		deadline,
		(jobStatus) -> jobStatus == JobStatus.FINISHED,
		TestingUtils.defaultScheduledExecutor());
	assertEquals(
		JobStatus.FINISHED,
		jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
}
 
Example 8
Source Project: flink   Source File: AbstractOperatorRestoreTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private void restoreJob(ClassLoader classLoader, ClusterClient<?> clusterClient, Deadline deadline, String savepointPath) throws Exception {
	JobGraph jobToRestore = createJobGraph(ExecutionMode.RESTORE);
	jobToRestore.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState));

	assertNotNull("Job doesn't have a JobID.", jobToRestore.getJobID());

	clusterClient.submitJob(jobToRestore, classLoader);

	CompletableFuture<JobStatus> jobStatusFuture = FutureUtils.retrySuccessfulWithDelay(
		() -> clusterClient.getJobStatus(jobToRestore.getJobID()),
		Time.milliseconds(50),
		deadline,
		(jobStatus) -> jobStatus == JobStatus.FINISHED,
		TestingUtils.defaultScheduledExecutor());
	assertEquals(
		JobStatus.FINISHED,
		jobStatusFuture.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS));
}
 
Example 9
Source Project: flink   Source File: FutureUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retry the given operation with the given delay in between successful completions where the
 * result does not match a given predicate.
 *
 * @param operation to retry
 * @param retryDelay delay between retries
 * @param deadline A deadline that specifies at what point we should stop retrying
 * @param acceptancePredicate Predicate to test whether the result is acceptable
 * @param scheduledExecutor executor to be used for the retry operation
 * @param <T> type of the result
 * @return Future which retries the given operation a given amount of times and delays the retry
 *   in case the predicate isn't matched
 */
public static <T> CompletableFuture<T> retrySuccessfulWithDelay(
	final Supplier<CompletableFuture<T>> operation,
	final Time retryDelay,
	final Deadline deadline,
	final Predicate<T> acceptancePredicate,
	final ScheduledExecutor scheduledExecutor) {

	final CompletableFuture<T> resultFuture = new CompletableFuture<>();

	retrySuccessfulOperationWithDelay(
		resultFuture,
		operation,
		retryDelay,
		deadline,
		acceptancePredicate,
		scheduledExecutor);

	return resultFuture;
}
 
Example 10
Source Project: flink   Source File: AbstractQueryableStateTestBase.java    License: Apache License 2.0 6 votes vote down vote up
private static <K, S extends State, V> CompletableFuture<S> getKvState(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	final CompletableFuture<S> resultFuture = new CompletableFuture<>();
	getKvStateIgnoringCertainExceptions(
			deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
			stateDescriptor, failForUnknownKeyOrNamespace, executor);
	return resultFuture;
}
 
Example 11
Source Project: flink   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Waits until all executions fulfill the given predicate.
 *
 * @param executionGraph for which to check the executions
 * @param executionPredicate predicate which is to be fulfilled
 * @param maxWaitMillis timeout for the wait operation
 * @throws TimeoutException if the executions did not reach the target state in time
 */
public static void waitForAllExecutionsPredicate(
		ExecutionGraph executionGraph,
		Predicate<AccessExecution> executionPredicate,
		long maxWaitMillis) throws TimeoutException {
	final Predicate<AccessExecutionGraph> allExecutionsPredicate = allExecutionsPredicate(executionPredicate);
	final Deadline deadline = Deadline.fromNow(Duration.ofMillis(maxWaitMillis));
	boolean predicateResult;

	do {
		predicateResult = allExecutionsPredicate.test(executionGraph);

		if (!predicateResult) {
			try {
				Thread.sleep(2L);
			} catch (InterruptedException ignored) {
				Thread.currentThread().interrupt();
			}
		}
	} while (!predicateResult && deadline.hasTimeLeft());

	if (!predicateResult) {
		throw new TimeoutException("Not all executions fulfilled the predicate in time.");
	}
}
 
Example 12
Source Project: flink   Source File: SystemProcessingTimeService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean shutdownServiceUninterruptible(long timeoutMs) {

	final Deadline deadline = Deadline.fromNow(Duration.ofMillis(timeoutMs));

	boolean shutdownComplete = false;
	boolean receivedInterrupt = false;

	do {
		try {
			// wait for a reasonable time for all pending timer threads to finish
			shutdownComplete = shutdownAndAwaitPending(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS);
		} catch (InterruptedException iex) {
			receivedInterrupt = true;
			LOG.trace("Intercepted attempt to interrupt timer service shutdown.", iex);
		}
	} while (deadline.hasTimeLeft() && !shutdownComplete);

	if (receivedInterrupt) {
		Thread.currentThread().interrupt();
	}

	return shutdownComplete;
}
 
Example 13
Source Project: flink   Source File: YARNITCase.java    License: Apache License 2.0 6 votes vote down vote up
private void waitApplicationFinishedElseKillIt(
		ApplicationId applicationId,
		Duration timeout,
		YarnClusterDescriptor yarnClusterDescriptor) throws Exception {
	Deadline deadline = Deadline.now().plus(timeout);
	YarnApplicationState state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();

	while (state != YarnApplicationState.FINISHED) {
		if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) {
			Assert.fail("Application became FAILED or KILLED while expecting FINISHED");
		}

		if (deadline.isOverdue()) {
			yarnClusterDescriptor.killCluster(applicationId);
			Assert.fail("Application didn't finish before timeout");
		}

		sleep(sleepIntervalInMS);
		state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();
	}
}
 
Example 14
Source Project: flink   Source File: MetricsAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static <X> X fetchMetric(final SupplierWithException<CompletableFuture<X>, IOException> clientOperation, final Predicate<X> predicate) throws InterruptedException, ExecutionException, TimeoutException {
	final CompletableFuture<X> responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> {
			try {
				return clientOperation.get();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		},
		Time.seconds(1),
		Deadline.fromNow(Duration.ofSeconds(5)),
		predicate,
		new ScheduledExecutorServiceAdapter(scheduledExecutorService));

	return responseFuture.get(30, TimeUnit.SECONDS);
}
 
Example 15
@Test
public void testMigrationAndRestore() throws Throwable {
	ClassLoader classLoader = this.getClass().getClassLoader();
	ClusterClient<?> clusterClient = MINI_CLUSTER_RESOURCE.getClusterClient();
	clusterClient.setDetached(true);
	final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);

	// submit job with old version savepoint and create a migrated savepoint in the new version
	String savepointPath = migrateJob(classLoader, clusterClient, deadline);
	// restore from migrated new version savepoint
	restoreJob(classLoader, clusterClient, deadline, savepointPath);
}
 
Example 16
Source Project: flink   Source File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static void waitUntilJobIsRunning(RestClusterClient<ApplicationId> restClusterClient, JobID jobId) throws Exception {
	CommonTestUtils.waitUntilCondition(
		() -> {
			final JobDetailsInfo jobDetails = restClusterClient.getJobDetails(jobId).get();
			return jobDetails.getJobVertexInfos()
				.stream()
				.map(toExecutionState())
				.allMatch(isRunning());
		},
		Deadline.fromNow(TIMEOUT));
}
 
Example 17
Source Project: Flink-CEPplus   Source File: SavepointITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void restoreJobAndVerifyState(String savepointPath, MiniClusterResourceFactory clusterFactory, int parallelism) throws Exception {
	final JobGraph jobGraph = createJobGraph(parallelism, 0, 1000);
	jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
	final JobID jobId = jobGraph.getJobID();
	StatefulCounter.resetForTest(parallelism);

	MiniClusterWithClientResource cluster = clusterFactory.get();
	cluster.before();
	ClusterClient<?> client = cluster.getClusterClient();

	try {
		client.setDetached(true);
		client.submitJob(jobGraph, SavepointITCase.class.getClassLoader());

		// Await state is restored
		StatefulCounter.getRestoreLatch().await();

		// Await some progress after restore
		StatefulCounter.getProgressLatch().await();

		client.cancel(jobId);

		FutureUtils.retrySuccessfulWithDelay(
			() -> client.getJobStatus(jobId),
			Time.milliseconds(50),
			Deadline.now().plus(Duration.ofSeconds(30)),
			status -> status == JobStatus.CANCELED,
			TestingUtils.defaultScheduledExecutor()
		);

		client.disposeSavepoint(savepointPath)
			.get();

		assertFalse("Savepoint not properly cleaned up.", new File(savepointPath).exists());
	} finally {
		cluster.after();
		StatefulCounter.resetForTest(parallelism);
	}
}
 
Example 18
private void waitUntilAllSlotsAreUsed(DispatcherGateway dispatcherGateway, Time timeout) throws ExecutionException, InterruptedException {
	FutureUtils.retrySuccessfulWithDelay(
		() -> dispatcherGateway.requestClusterOverview(timeout),
		Time.milliseconds(50L),
		Deadline.fromNow(Duration.ofMillis(timeout.toMilliseconds())),
		clusterOverview -> clusterOverview.getNumTaskManagersConnected() >= 1 &&
			clusterOverview.getNumSlotsAvailable() == 0 &&
			clusterOverview.getNumSlotsTotal() == 2,
		TestingUtils.defaultScheduledExecutor())
		.get();
}
 
Example 19
Source Project: flink   Source File: LocalStandaloneKafkaResource.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<String> readMessage(int expectedNumMessages, String groupId, String topic) throws IOException {
	final List<String> messages = Collections.synchronizedList(new ArrayList<>(
			expectedNumMessages));

	try (final AutoClosableProcess kafka = AutoClosableProcess
		.create(kafkaDir.resolve(Paths.get("bin", "kafka-console-consumer.sh")).toString(),
			"--bootstrap-server",
			KAFKA_ADDRESS,
			"--from-beginning",
			"--max-messages",
			String.valueOf(expectedNumMessages),
			"--topic",
			topic,
			"--consumer-property",
			"group.id=" + groupId)
		.setStdoutProcessor(messages::add)
		.runNonBlocking()) {

		final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(120));
		while (deadline.hasTimeLeft() && messages.size() < expectedNumMessages) {
			try {
				LOG.info("Waiting for messages. Received {}/{}.", messages.size(),
						expectedNumMessages);
				Thread.sleep(500);
			} catch (InterruptedException e) {
				Thread.currentThread().interrupt();
				break;
			}
		}
		if (messages.size() != expectedNumMessages) {
			throw new IOException("Could not read expected number of messages.");
		}
		return messages;
	}
}
 
Example 20
Source Project: flink   Source File: AbstractQueryableStateTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private static <K, S extends State, V> void getKvStateIgnoringCertainExceptions(
		final Deadline deadline,
		final CompletableFuture<S> resultFuture,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	if (!resultFuture.isDone()) {
		CompletableFuture<S> expected = client.getKvState(jobId, queryName, key, keyTypeInfo, stateDescriptor);
		expected.whenCompleteAsync((result, throwable) -> {
			if (throwable != null) {
				if (
						throwable.getCause() instanceof CancellationException ||
						throwable.getCause() instanceof AssertionError ||
						(failForUnknownKeyOrNamespace && throwable.getCause() instanceof UnknownKeyOrNamespaceException)
				) {
					resultFuture.completeExceptionally(throwable.getCause());
				} else if (deadline.hasTimeLeft()) {
					getKvStateIgnoringCertainExceptions(
							deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
							stateDescriptor, failForUnknownKeyOrNamespace, executor);
				}
			} else {
				resultFuture.complete(result);
			}
		}, executor);

		resultFuture.whenComplete((result, throwable) -> expected.cancel(false));
	}
}
 
Example 21
Source Project: Flink-CEPplus   Source File: TaskExecutorITCase.java    License: Apache License 2.0 5 votes vote down vote up
private CompletableFuture<JobResult> submitJobAndWaitUntilRunning(JobGraph jobGraph) throws Exception {
	miniCluster.submitJob(jobGraph).get();

	final CompletableFuture<JobResult> jobResultFuture = miniCluster.requestJobResult(jobGraph.getJobID());

	assertThat(jobResultFuture.isDone(), is(false));

	CommonTestUtils.waitUntilCondition(
		jobIsRunning(() -> miniCluster.getExecutionGraph(jobGraph.getJobID())),
		Deadline.fromNow(TESTING_TIMEOUT),
		50L);

	return jobResultFuture;
}
 
Example 22
Source Project: Flink-CEPplus   Source File: MetricsAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
private static <X> X fetchMetric(final SupplierWithException<CompletableFuture<X>, IOException> clientOperation, final Predicate<X> predicate) throws InterruptedException, ExecutionException, TimeoutException {
	final CompletableFuture<X> responseFuture = FutureUtils.retrySuccessfulWithDelay(() -> {
			try {
				return clientOperation.get();
			} catch (IOException e) {
				throw new RuntimeException(e);
			}
		},
		Time.seconds(1),
		Deadline.fromNow(Duration.ofSeconds(5)),
		predicate,
		new ScheduledExecutorServiceAdapter(scheduledExecutorService));

	return responseFuture.get(30, TimeUnit.SECONDS);
}
 
Example 23
/**
 * Tests simple value state queryable state instance. Each source emits
 * (subtaskIndex, 0)..(subtaskIndex, numElements) tuples, which are then
 * queried. The tests succeeds after each subtask index is queried with
 * value numElements (the latest element updated the state).
 */
@Test
public void testValueState() throws Exception {
	final Deadline deadline = Deadline.now().plus(TEST_TIMEOUT);
	final long numElements = 1024L;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setStateBackend(stateBackend);
	env.setParallelism(maxParallelism);
	// Very important, because cluster is shared between tests and we
	// don't explicitly check that all slots are available before
	// submitting.
	env.setRestartStrategy(RestartStrategies.fixedDelayRestart(Integer.MAX_VALUE, 1000L));

	DataStream<Tuple2<Integer, Long>> source = env.addSource(new TestAscendingValueSource(numElements));

	// Value state
	ValueStateDescriptor<Tuple2<Integer, Long>> valueState = new ValueStateDescriptor<>("any", source.getType());

	source.keyBy(new KeySelector<Tuple2<Integer, Long>, Integer>() {
		private static final long serialVersionUID = 7662520075515707428L;

		@Override
		public Integer getKey(Tuple2<Integer, Long> value) {
			return value.f0;
		}
	}).asQueryableState("hakuna", valueState);

	try (AutoCancellableJob autoCancellableJob = new AutoCancellableJob(deadline, clusterClient, env)) {

		final JobID jobId = autoCancellableJob.getJobId();
		final JobGraph jobGraph = autoCancellableJob.getJobGraph();

		clusterClient.setDetached(true);
		clusterClient.submitJob(jobGraph, AbstractQueryableStateTestBase.class.getClassLoader());

		executeValueQuery(deadline, client, jobId, "hakuna", valueState, numElements);
	}
}
 
Example 24
Source Project: beam   Source File: FlinkPortableClientEntryPoint.java    License: Apache License 2.0 5 votes vote down vote up
private void startJobService() throws Exception {
  jobInvokerFactory = new DetachedJobInvokerFactory();
  jobServer =
      FlinkJobServerDriver.fromConfig(
          FlinkJobServerDriver.parseArgs(
              new String[] {"--job-port=" + jobPort, "--artifact-port=0", "--expansion-port=0"}),
          jobInvokerFactory);
  jobServerThread = new Thread(jobServer);
  jobServerThread.start();

  Deadline deadline = Deadline.fromNow(JOB_SERVICE_STARTUP_TIMEOUT);
  while (jobServer.getJobServerUrl() == null && deadline.hasTimeLeft()) {
    try {
      Thread.sleep(500);
    } catch (InterruptedException interruptEx) {
      Thread.currentThread().interrupt();
      throw new RuntimeException(interruptEx);
    }
  }

  if (!jobServerThread.isAlive()) {
    throw new IllegalStateException("Job service thread is not alive");
  }

  if (jobServer.getJobServerUrl() == null) {
    String msg = String.format("Timeout of %s waiting for job service to start.", deadline);
    throw new TimeoutException(msg);
  }
}
 
Example 25
Source Project: flink   Source File: SQLClientKafkaITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void checkCsvResultFile() throws Exception {
	boolean success = false;
	final Deadline deadline = Deadline.fromNow(Duration.ofSeconds(120));
	while (deadline.hasTimeLeft()) {
		if (Files.exists(result)) {
			List<String> lines = readCsvResultFiles(result);
			if (lines.size() == 4) {
				success = true;
				assertThat(
					lines.toArray(new String[0]),
					arrayContainingInAnyOrder(
						"2018-03-12 08:00:00.000,Alice,This was a warning.,2,Success constant folding.",
						"2018-03-12 09:00:00.000,Bob,This was another warning.,1,Success constant folding.",
						"2018-03-12 09:00:00.000,Steve,This was another info.,2,Success constant folding.",
						"2018-03-12 09:00:00.000,Alice,This was a info.,1,Success constant folding."
					)
				);
				break;
			} else {
				LOG.info("The target CSV {} does not contain enough records, current {} records, left time: {}s",
					result, lines.size(), deadline.timeLeft().getSeconds());
			}
		} else {
			LOG.info("The target CSV {} does not exist now", result);
		}
		Thread.sleep(500);
	}
	Assert.assertTrue("Did not get expected results before timeout.", success);
}
 
Example 26
private static <K, S extends State, V> void getKvStateIgnoringCertainExceptions(
		final Deadline deadline,
		final CompletableFuture<S> resultFuture,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryName,
		final K key,
		final TypeInformation<K> keyTypeInfo,
		final StateDescriptor<S, V> stateDescriptor,
		final boolean failForUnknownKeyOrNamespace,
		final ScheduledExecutor executor) {

	if (!resultFuture.isDone()) {
		CompletableFuture<S> expected = client.getKvState(jobId, queryName, key, keyTypeInfo, stateDescriptor);
		expected.whenCompleteAsync((result, throwable) -> {
			if (throwable != null) {
				if (
						throwable.getCause() instanceof CancellationException ||
						throwable.getCause() instanceof AssertionError ||
						(failForUnknownKeyOrNamespace && throwable.getCause() instanceof UnknownKeyOrNamespaceException)
				) {
					resultFuture.completeExceptionally(throwable.getCause());
				} else if (deadline.hasTimeLeft()) {
					getKvStateIgnoringCertainExceptions(
							deadline, resultFuture, client, jobId, queryName, key, keyTypeInfo,
							stateDescriptor, failForUnknownKeyOrNamespace, executor);
				}
			} else {
				resultFuture.complete(result);
			}
		}, executor);

		resultFuture.whenComplete((result, throwable) -> expected.cancel(false));
	}
}
 
Example 27
/**
 * Retry a query for state for keys between 0 and {@link #maxParallelism} until
 * <tt>expected</tt> equals the value of the result tuple's second field.
 */
private void executeValueQuery(
		final Deadline deadline,
		final QueryableStateClient client,
		final JobID jobId,
		final String queryableStateName,
		final ValueStateDescriptor<Tuple2<Integer, Long>> stateDescriptor,
		final long expected) throws Exception {

	for (int key = 0; key < maxParallelism; key++) {
		boolean success = false;
		while (deadline.hasTimeLeft() && !success) {
			CompletableFuture<ValueState<Tuple2<Integer, Long>>> future = getKvState(
					deadline,
					client,
					jobId,
					queryableStateName,
					key,
					BasicTypeInfo.INT_TYPE_INFO,
					stateDescriptor,
					false,
					executor);

			Tuple2<Integer, Long> value = future.get(deadline.timeLeft().toMillis(), TimeUnit.MILLISECONDS).value();

			assertEquals("Key mismatch", key, value.f0.intValue());
			if (expected == value.f1) {
				success = true;
			} else {
				// Retry
				Thread.sleep(RETRY_TIMEOUT);
			}
		}

		assertTrue("Did not succeed query", success);
	}
}
 
Example 28
Source Project: flink   Source File: YarnTestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws Exception {
	Deadline deadline = Deadline.now().plus(Duration.ofSeconds(10));

	boolean isAnyJobRunning = yarnClient.getApplications().stream()
		.anyMatch(YarnTestBase::isApplicationRunning);

	while (deadline.hasTimeLeft() && isAnyJobRunning) {
		try {
			Thread.sleep(500);
		} catch (InterruptedException e) {
			Assert.fail("Should not happen");
		}
		isAnyJobRunning = yarnClient.getApplications().stream()
			.anyMatch(YarnTestBase::isApplicationRunning);
	}

	if (isAnyJobRunning) {
		final List<String> runningApps = yarnClient.getApplications().stream()
			.filter(YarnTestBase::isApplicationRunning)
			.map(app -> "App " + app.getApplicationId() + " is in state " + app.getYarnApplicationState() + '.')
			.collect(Collectors.toList());
		if (!runningApps.isEmpty()) {
			Assert.fail("There is at least one application on the cluster that is not finished." + runningApps);
		}
	}
}
 
Example 29
Source Project: Flink-CEPplus   Source File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void waitForApplicationAttempt(final ApplicationId applicationId, final int attemptId) throws Exception {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null, "yarnClient must be initialized");

	CommonTestUtils.waitUntilCondition(() -> {
		final ApplicationReport applicationReport = yarnClient.getApplicationReport(applicationId);
		return applicationReport.getCurrentApplicationAttemptId().getAttemptId() >= attemptId;
	}, Deadline.fromNow(TIMEOUT));
}
 
Example 30
Source Project: Flink-CEPplus   Source File: YARNHighAvailabilityITCase.java    License: Apache License 2.0 5 votes vote down vote up
private void killApplicationAndWait(final ApplicationId id) throws Exception {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null, "yarnClient must be initialized");

	yarnClient.killApplication(id);

	CommonTestUtils.waitUntilCondition(() -> !yarnClient.getApplications(EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FINISHED)).isEmpty(),
		Deadline.fromNow(TIMEOUT));
}