Java Code Examples for org.apache.flink.core.testutils.OneShotLatch

The following examples show how to use org.apache.flink.core.testutils.OneShotLatch. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: CliFrontendStopWithSavepointTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStopWithMaxWMAndDefaultSavepointDir() throws Exception {
	JobID jid = new JobID();

	String[] parameters = { "-p", "-d", jid.toString() };
	OneShotLatch stopWithSavepointLatch = new OneShotLatch();
	TestingClusterClient<String> clusterClient = new TestingClusterClient<>();
	clusterClient.setStopWithSavepointFunction((jobID, advanceToEndOfEventTime, savepointDirectory) -> {
		assertThat(jobID, is(jid));
		assertThat(advanceToEndOfEventTime, is(true));
		assertNull(savepointDirectory);
		stopWithSavepointLatch.trigger();
		return CompletableFuture.completedFuture(savepointDirectory);
	});
	MockedCliFrontend testFrontend = new MockedCliFrontend(clusterClient);
	testFrontend.stop(parameters);

	stopWithSavepointLatch.await();
}
 
Example 2
Source Project: flink   Source File: CliFrontendCancelTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCancel() throws Exception {
	// test cancel properly
	JobID jid = new JobID();

	OneShotLatch cancelLatch = new OneShotLatch();

	String[] parameters = { jid.toString() };

	TestingClusterClient<String> clusterClient = new TestingClusterClient<>();

	clusterClient.setCancelFunction(jobID -> {
		cancelLatch.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());
	});

	MockedCliFrontend testFrontend = new MockedCliFrontend(clusterClient);
	testFrontend.cancel(parameters);
	cancelLatch.await();
}
 
Example 3
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCancellationFailsWithBlockingLock() throws Exception {
	syncLatch = new OneShotLatch();

	StreamConfig cfg = new StreamConfig(new Configuration());
	Task task = createTask(CancelFailingTask.class, cfg, new Configuration());

	// start the task and wait until it runs
	// execution state RUNNING is not enough, we need to wait until the stream task's run() method
	// is entered
	task.startTaskThread();
	syncLatch.await();

	// cancel the execution - this should lead to smooth shutdown
	task.cancelExecution();
	task.getExecutingThread().join();

	assertEquals(ExecutionState.CANCELED, task.getExecutionState());
}
 
Example 4
Source Project: flink   Source File: BlockingCheckpointOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 
Example 5
Source Project: flink   Source File: BlockingFSDataInputStream.java    License: Apache License 2.0 6 votes vote down vote up
public BlockingFSDataInputStream(
	@Nullable FSDataInputStream delegate,
	@Nullable OneShotLatch waitForBlock,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitUntilStreamBlocked = waitForBlock;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 
Example 6
Source Project: flink   Source File: JobManagerRunnerImplTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLibraryCacheManagerRegistration() throws Exception {
	final OneShotLatch registerClassLoaderLatch = new OneShotLatch();
	final OneShotLatch closeClassLoaderLeaseLatch = new OneShotLatch();
	final TestingClassLoaderLease classLoaderLease = TestingClassLoaderLease.newBuilder()
		.setGetOrResolveClassLoaderFunction((permanentBlobKeys, urls) -> {
			registerClassLoaderLatch.trigger();
			return JobManagerRunnerImplTest.class.getClassLoader();
		})
		.setCloseRunnable(closeClassLoaderLeaseLatch::trigger)
		.build();
	final JobManagerRunner jobManagerRunner = createJobManagerRunner(classLoaderLease);

	try {
		jobManagerRunner.start();

		registerClassLoaderLatch.await();

		jobManagerRunner.close();

		closeClassLoaderLeaseLatch.await();
	} finally {
		jobManagerRunner.close();
	}
}
 
Example 7
Source Project: Flink-CEPplus   Source File: FutureUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRunAfterwards() throws Exception {
	final CompletableFuture<Void> inputFuture = new CompletableFuture<>();
	final OneShotLatch runnableLatch = new OneShotLatch();

	final CompletableFuture<Void> runFuture = FutureUtils.runAfterwards(
		inputFuture,
		runnableLatch::trigger);

	assertThat(runnableLatch.isTriggered(), is(false));
	assertThat(runFuture.isDone(), is(false));

	inputFuture.complete(null);

	assertThat(runnableLatch.isTriggered(), is(true));
	assertThat(runFuture.isDone(), is(true));

	// check that this future is not exceptionally completed
	runFuture.get();
}
 
Example 8
Source Project: flink   Source File: AsyncCallsTest.java    License: Apache License 2.0 6 votes vote down vote up
private FencedTestEndpoint(
		RpcService rpcService,
		ReentrantLock lock,
		AtomicBoolean concurrentAccess,
		UUID initialFencingToken,
		OneShotLatch enteringSetNewFencingToken,
		OneShotLatch triggerSetNewFencingToken) {
	super(rpcService);

	this.lock = lock;
	this.concurrentAccess = concurrentAccess;

	this.enteringSetNewFencingToken = enteringSetNewFencingToken;
	this.triggerSetNewFencingToken = triggerSetNewFencingToken;

	// make it look as if we are running in the main thread
	currentMainThread.set(Thread.currentThread());

	try {
		setFencingToken(initialFencingToken);
	} finally {
		currentMainThread.set(null);
	}
}
 
Example 9
Source Project: flink   Source File: FutureUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testComposeAfterwards() throws ExecutionException, InterruptedException {
	final CompletableFuture<Void> inputFuture = new CompletableFuture<>();
	final OneShotLatch composeLatch = new OneShotLatch();

	final CompletableFuture<Void> composeFuture = FutureUtils.composeAfterwards(
		inputFuture,
		() -> {
			composeLatch.trigger();
			return CompletableFuture.completedFuture(null);
		});

	assertThat(composeLatch.isTriggered(), is(false));
	assertThat(composeFuture.isDone(), is(false));

	inputFuture.complete(null);

	assertThat(composeLatch.isTriggered(), is(true));
	assertThat(composeFuture.isDone(), is(true));

	// check that tthis future is not exceptionally completed
	composeFuture.get();
}
 
Example 10
public BlockingCheckpointOutputStream(
	@Nullable FSDataOutputStream delegate,
	@Nullable OneShotLatch waitForBlocking,
	@Nullable OneShotLatch triggerUnblock,
	long blockAtPosition) {

	this.delegate = delegate;
	this.triggerUnblock = triggerUnblock;
	this.waitForBlocking = waitForBlocking;
	this.blockAtPosition = blockAtPosition;
	if (delegate != null) {
		try {
			this.position = delegate.getPos();
		} catch (IOException e) {
			throw new RuntimeException(e);
		}
	} else {
		this.position = 0;
	}
	this.closed = new AtomicBoolean(false);
}
 
Example 11
Source Project: flink   Source File: KubernetesResourceManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
private FlinkKubeClient createTestingFlinkKubeClientAllocatingPodsAfter(
		int numberOfRetries,
		AtomicInteger retries,
		OneShotLatch podCreated) {
	ExecutorService kubeClientExecutorService = Executors.newDirectExecutorService();
	return new Fabric8FlinkKubeClient(flinkConfig, kubeClient, () -> kubeClientExecutorService) {
		@Override
		public CompletableFuture<Void> createTaskManagerPod(KubernetesPod kubernetesPod) {
			if (retries.getAndIncrement() < numberOfRetries) {
				return FutureUtils.completedExceptionally(new RuntimeException("Exception"));
			}
			podCreated.trigger();
			return super.createTaskManagerPod(kubernetesPod);
		}
	};
}
 
Example 12
Source Project: flink   Source File: WaitingSource.java    License: Apache License 2.0 5 votes vote down vote up
public WaitingSource(SourceFunction<T> source, TypeInformation<T> returnType) {
	this.source = source;
	this.returnType = returnType;
	this.guardId = UUID.randomUUID().toString();

	guards.put(guardId, new OneShotLatch());
	this.running = true;
}
 
Example 13
Source Project: flink   Source File: NetworkBufferPoolTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests {@link NetworkBufferPool#requestMemorySegments()}, verifying it may be aborted in
 * case of a concurrent {@link NetworkBufferPool#destroy()} call.
 */
@Test
public void testRequestMemorySegmentsInterruptable() throws Exception {
	final int numBuffers = 10;

	NetworkBufferPool globalPool = new NetworkBufferPool(numBuffers, 128, 10);
	MemorySegment segment = globalPool.requestMemorySegment();
	assertNotNull(segment);

	final OneShotLatch isRunning = new OneShotLatch();
	CheckedThread asyncRequest = new CheckedThread() {
		@Override
		public void go() throws Exception {
			isRunning.trigger();
			globalPool.requestMemorySegments();
		}
	};
	asyncRequest.start();

	// We want the destroy call inside the blocking part of the globalPool.requestMemorySegments()
	// call above. We cannot guarantee this though but make it highly probable:
	isRunning.await();
	Thread.sleep(10);
	globalPool.destroy();

	segment.free();

	expectedException.expect(IllegalStateException.class);
	expectedException.expectMessage("destroyed");
	try {
		asyncRequest.sync();
	} finally {
		globalPool.destroy();
	}
}
 
Example 14
Source Project: flink   Source File: SessionDispatcherLeaderProcessTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void closeAsync_duringJobRecovery_preventsDispatcherServiceCreation() throws Exception {
	final OneShotLatch jobRecoveryStartedLatch = new OneShotLatch();
	final OneShotLatch completeJobRecoveryLatch = new OneShotLatch();
	final OneShotLatch createDispatcherServiceLatch = new OneShotLatch();

	this.jobGraphStore = TestingJobGraphStore.newBuilder()
		.setJobIdsFunction(storedJobs -> {
			jobRecoveryStartedLatch.trigger();
			completeJobRecoveryLatch.await();
			return storedJobs;
		})
		.build();

	this.dispatcherServiceFactory = TestingDispatcherServiceFactory.newBuilder()
		.setCreateFunction(
			(ignoredA, ignoredB, ignoredC) -> {
				createDispatcherServiceLatch.trigger();
				return TestingDispatcherGatewayService.newBuilder().build();
			})
		.build();

	try (final SessionDispatcherLeaderProcess dispatcherLeaderProcess = createDispatcherLeaderProcess()) {
		dispatcherLeaderProcess.start();

		jobRecoveryStartedLatch.await();

		dispatcherLeaderProcess.closeAsync();

		completeJobRecoveryLatch.trigger();

		try {
			createDispatcherServiceLatch.await(10L, TimeUnit.MILLISECONDS);
			fail("No dispatcher service should be created after the process has been stopped.");
		} catch (TimeoutException expected) {}
	}
}
 
Example 15
FileVerifyingSourceContext(OneShotLatch latch,
						ContinuousFileMonitoringFunction src,
						int elementsBeforeNotifying,
						int elementsBeforeCanceling) {
	this.latch = latch;
	this.seenFiles = new TreeSet<>();
	this.src = src;
	this.elementsBeforeNotifying = elementsBeforeNotifying;
	this.elementsBeforeCanceling = elementsBeforeCanceling;
}
 
Example 16
Source Project: flink   Source File: KafkaConsumerThreadTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static TestConsumer createMockConsumer(
		final Map<TopicPartition, Long> mockConsumerAssignmentAndPosition,
		final Map<TopicPartition, Long> mockRetrievedPositions,
		final boolean earlyWakeup,
		final OneShotLatch midAssignmentLatch,
		final OneShotLatch continueAssignmentLatch) {

	return new TestConsumer(mockConsumerAssignmentAndPosition, mockRetrievedPositions, earlyWakeup, midAssignmentLatch, continueAssignmentLatch);
}
 
Example 17
Source Project: pulsar-flink   Source File: PulsarFetcherTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentPartitionsDiscoveryAndLoopFetching() throws Exception {
    String tp = topicName("test", 2);
    TestSourceContext<Long> sourceContext = new TestSourceContext<Long>();
    Map<String, MessageId> offset = Collections.singletonMap(topicName(tp, 1), MessageId.latest);

    OneShotLatch fetchLoopWaitLatch = new OneShotLatch();
    OneShotLatch stateIterationBlockLatch = new OneShotLatch();

    TestFetcher fetcher = new TestFetcher(
            sourceContext,
            offset,
            null,
            null,
            new TestProcessingTimeService(),
            10,
            fetchLoopWaitLatch,
            stateIterationBlockLatch);

    // ----- run the fetcher -----

    final CheckedThread checkedThread = new CheckedThread() {
        @Override
        public void go() throws Exception {
            fetcher.runFetchLoop();
        }
    };
    checkedThread.start();

    fetchLoopWaitLatch.await();
    fetcher.addDiscoveredTopics(Sets.newSet(tp));

    stateIterationBlockLatch.trigger();
    checkedThread.sync();
}
 
Example 18
Source Project: flink   Source File: AsyncCallsTest.java    License: Apache License 2.0 5 votes vote down vote up
private FencedTestEndpoint(
		RpcService rpcService,
		ReentrantLock lock,
		AtomicBoolean concurrentAccess,
		UUID initialFencingToken,
		OneShotLatch enteringSetNewFencingToken,
		OneShotLatch triggerSetNewFencingToken) {
	super(rpcService, initialFencingToken);

	this.lock = lock;
	this.concurrentAccess = concurrentAccess;

	this.enteringSetNewFencingToken = enteringSetNewFencingToken;
	this.triggerSetNewFencingToken = triggerSetNewFencingToken;
}
 
Example 19
Source Project: Flink-CEPplus   Source File: AsyncCallsTest.java    License: Apache License 2.0 5 votes vote down vote up
protected FencedTestEndpoint(
		RpcService rpcService,
		ReentrantLock lock,
		AtomicBoolean concurrentAccess) {
	this(
		rpcService,
		lock,
		concurrentAccess,
		UUID.randomUUID(),
		new OneShotLatch(),
		new OneShotLatch());
}
 
Example 20
Source Project: flink   Source File: TestableKinesisDataFetcher.java    License: Apache License 2.0 5 votes vote down vote up
public TestableKinesisDataFetcher(
		List<String> fakeStreams,
		SourceFunction.SourceContext<T> sourceContext,
		Properties fakeConfiguration,
		KinesisDeserializationSchema<T> deserializationSchema,
		int fakeTotalCountOfSubtasks,
		int fakeIndexOfThisSubtask,
		AtomicReference<Throwable> thrownErrorUnderTest,
		LinkedList<KinesisStreamShardState> subscribedShardsStateUnderTest,
		HashMap<String, String> subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		KinesisProxyInterface fakeKinesis) {
	super(
		fakeStreams,
		sourceContext,
		sourceContext.getCheckpointLock(),
		getMockedRuntimeContext(fakeTotalCountOfSubtasks, fakeIndexOfThisSubtask),
		fakeConfiguration,
		deserializationSchema,
		DEFAULT_SHARD_ASSIGNER,
		null,
		null,
		thrownErrorUnderTest,
		subscribedShardsStateUnderTest,
		subscribedStreamsToLastDiscoveredShardIdsStateUnderTest,
		(properties) -> fakeKinesis);

	this.runWaiter = new OneShotLatch();
	this.initialDiscoveryWaiter = new OneShotLatch();
	this.shutdownWaiter = new OneShotLatch();

	this.running = true;
}
 
Example 21
Source Project: Flink-CEPplus   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void run() throws Exception {
	final OneShotLatch latch = new OneShotLatch();
	final Object lock = new Object();

	LockHolder holder = new LockHolder(lock, latch);
	holder.start();
	try {
		// cancellation should try and cancel this
		getCancelables().registerCloseable(holder);

		// wait till the lock holder has the lock
		latch.await();

		// we are at the point where cancelling can happen
		syncLatch.trigger();

		// try to acquire the lock - this is not possible as long as the lock holder
		// thread lives
		//noinspection SynchronizationOnLocalVariableOrMethodParameter
		synchronized (lock) {
			// nothing
		}
	}
	finally {
		holder.close();
	}

}
 
Example 22
Source Project: flink   Source File: StreamTaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void performDefaultAction(DefaultActionContext context) throws Exception {
	final OneShotLatch latch = new OneShotLatch();
	final Object lock = new Object();

	LockHolder holder = new LockHolder(lock, latch);
	holder.start();
	try {
		// cancellation should try and cancel this
		getCancelables().registerCloseable(holder);

		// wait till the lock holder has the lock
		latch.await();

		// we are at the point where cancelling can happen
		syncLatch.trigger();

		// try to acquire the lock - this is not possible as long as the lock holder
		// thread lives
		//noinspection SynchronizationOnLocalVariableOrMethodParameter
		synchronized (lock) {
			// nothing
		}
	}
	finally {
		holder.close();
	}
	context.allActionsCompleted();
}
 
Example 23
Source Project: Flink-CEPplus   Source File: AkkaRpcServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testScheduleRunnable() throws Exception {
	final OneShotLatch latch = new OneShotLatch();
	final long delay = 100L;
	final long start = System.nanoTime();

	ScheduledFuture<?> scheduledFuture = akkaRpcService.scheduleRunnable(latch::trigger, delay, TimeUnit.MILLISECONDS);

	scheduledFuture.get();

	assertTrue(latch.isTriggered());
	final long stop = System.nanoTime();

	assertTrue("call was not properly delayed", ((stop - start) / 1000000) >= delay);
}
 
Example 24
Source Project: flink   Source File: FutureUtilsTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testComposeAfterwardsFirstExceptional() throws InterruptedException {
	final CompletableFuture<Void> inputFuture = new CompletableFuture<>();
	final OneShotLatch composeLatch = new OneShotLatch();
	final FlinkException testException = new FlinkException("Test exception");

	final CompletableFuture<Void> composeFuture = FutureUtils.composeAfterwards(
		inputFuture,
		() -> {
			composeLatch.trigger();
			return CompletableFuture.completedFuture(null);
		});

	assertThat(composeLatch.isTriggered(), is(false));
	assertThat(composeFuture.isDone(), is(false));

	inputFuture.completeExceptionally(testException);

	assertThat(composeLatch.isTriggered(), is(true));
	assertThat(composeFuture.isDone(), is(true));

	// check that this future is not exceptionally completed
	try {
		composeFuture.get();
		fail("Expected an exceptional completion");
	} catch (ExecutionException ee) {
		assertThat(ExceptionUtils.stripExecutionException(ee), is(testException));
	}
}
 
Example 25
Source Project: flink   Source File: InitOutputPathTest.java    License: Apache License 2.0 5 votes vote down vote up
SyncedFileSystem(
		OneShotLatch deleteTriggerLatch,
		OneShotLatch mkdirsTriggerLatch,
		OneShotLatch deleteAwaitLatch,
		OneShotLatch mkdirsAwaitLatch) {

	this.deleteTriggerLatch = deleteTriggerLatch;
	this.mkdirsTriggerLatch = mkdirsTriggerLatch;
	this.deleteAwaitLatch = deleteAwaitLatch;
	this.mkdirsAwaitLatch = mkdirsAwaitLatch;
}
 
Example 26
Source Project: Flink-CEPplus   Source File: AsyncWaitOperatorTest.java    License: Apache License 2.0 5 votes vote down vote up
public TestAsyncWaitOperator(
		AsyncFunction<IN, OUT> asyncFunction,
		long timeout,
		int capacity,
		AsyncDataStream.OutputMode outputMode,
		OneShotLatch closingLatch) {
	super(asyncFunction, timeout, capacity, outputMode);

	this.closingLatch = Preconditions.checkNotNull(closingLatch);
}
 
Example 27
Source Project: Flink-CEPplus   Source File: TaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
private AllocateSlotNotifyingTaskSlotTable(
		Collection<ResourceProfile> resourceProfiles,
		TimerService<AllocationID> timerService,
		OneShotLatch allocateSlotLatch) {
	super(resourceProfiles, timerService);
	this.allocateSlotLatch = allocateSlotLatch;
}
 
Example 28
Source Project: flink   Source File: TaskTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() {
	awaitLatch = new OneShotLatch();
	triggerLatch = new OneShotLatch();

	shuffleEnvironment = new NettyShuffleEnvironmentBuilder().build();
}
 
Example 29
Source Project: flink   Source File: StatefulSequenceSourceTest.java    License: Apache License 2.0 5 votes vote down vote up
public BlockingSourceContext(String name, OneShotLatch latchToTrigger, OneShotLatch latchToWait,
							ConcurrentHashMap<String, List<T>> output, int elemToFire) {
	this.name = name;
	this.lock = new Object();
	this.latchToTrigger = latchToTrigger;
	this.latchToWait = latchToWait;
	this.collector = output;
	this.threshold = elemToFire;

	this.localOutput = new ArrayList<>();
	List<T> prev = collector.put(name, localOutput);
	if (prev != null) {
		Assert.fail();
	}
}
 
Example 30
Source Project: flink   Source File: SystemProcessingTimeServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
private static SystemProcessingTimeService createBlockingSystemProcessingTimeService(
		final OneShotLatch blockUntilTriggered,
		final AtomicBoolean check) {

	final OneShotLatch waitUntilTimerStarted = new OneShotLatch();

	Preconditions.checkState(!check.get());

	final SystemProcessingTimeService timeService = new SystemProcessingTimeService(exception -> {});

	timeService.scheduleAtFixedRate(
		timestamp -> {

			waitUntilTimerStarted.trigger();

			boolean unblocked = false;

			while (!unblocked) {
				try {
					blockUntilTriggered.await();
					unblocked = true;
				} catch (InterruptedException ignore) {
				}
			}

			check.set(true);
		},
		0L,
		10L);

	try {
		waitUntilTimerStarted.await();
	} catch (InterruptedException e) {
		Assert.fail("Problem while starting up service.");
	}

	return timeService;
}