org.apache.flink.core.testutils.BlockerSync Java Examples

The following examples show how to use org.apache.flink.core.testutils.BlockerSync. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NettyShuffleEnvironmentTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSlowIODoesNotBlockRelease() throws Exception {
	BlockerSync sync = new BlockerSync();
	ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {
		@Override
		public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
			sync.blockNonInterruptible();
			super.releasePartition(partitionId, cause);
		}
	};

	NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder()
		.setResultPartitionManager(blockingResultPartitionManager)
		.setIoExecutor(Executors.newFixedThreadPool(1))
		.build();

	shuffleEnvironment.releasePartitionsLocally(Collections.singleton(new ResultPartitionID()));
	sync.awaitBlocker();
	sync.releaseBlocker();
}
 
Example #2
Source File: KryoSerializerConcurrencyTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #3
Source File: AbstractMetricGroupTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 
Example #4
Source File: AvroSerializerConcurrencyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #5
Source File: AbstractMetricGroupTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 
Example #6
Source File: KryoSerializerConcurrencyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #7
Source File: AvroSerializerConcurrencyTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #8
Source File: AbstractMetricGroupTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetAllVariablesDoesNotDeadlock() throws InterruptedException {
	final TestMetricRegistry registry = new TestMetricRegistry();

	final MetricGroup parent = new GenericMetricGroup(registry, UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(), "parent");
	final MetricGroup child = parent.addGroup("child");

	final Thread parentRegisteringThread = new Thread(() -> parent.counter("parent_counter"));
	final Thread childRegisteringThread = new Thread(() -> child.counter("child_counter"));

	final BlockerSync parentSync = new BlockerSync();
	final BlockerSync childSync = new BlockerSync();

	try {
		// start both threads and have them block in the registry, so they acquire the lock of their respective group
		registry.setOnRegistrationAction(childSync::blockNonInterruptible);
		childRegisteringThread.start();
		childSync.awaitBlocker();

		registry.setOnRegistrationAction(parentSync::blockNonInterruptible);
		parentRegisteringThread.start();
		parentSync.awaitBlocker();

		// the parent thread remains blocked to simulate the child thread holding some lock in the registry/reporter
		// the child thread continues execution and calls getAllVariables()
		// in the past this would block indefinitely since the method acquires the locks of all parent groups
		childSync.releaseBlocker();
		// wait with a timeout to ensure the finally block is executed _at some point_, un-blocking the parent
		childRegisteringThread.join(1000 * 10);

		parentSync.releaseBlocker();
		parentRegisteringThread.join();
	} finally {
		parentSync.releaseBlocker();
		childSync.releaseBlocker();
		parentRegisteringThread.join();
		childRegisteringThread.join();
	}
}
 
Example #9
Source File: AvroSerializerConcurrencyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final AvroSerializer<String> serializer = new AvroSerializer<>(String.class);

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #10
Source File: KryoSerializerConcurrencyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testConcurrentUseOfSerializer() throws Exception {
	final KryoSerializer<String> serializer = new KryoSerializer<>(String.class, new ExecutionConfig());

	final BlockerSync sync = new BlockerSync();

	final DataOutputView regularOut = new DataOutputSerializer(32);
	final DataOutputView lockingOut = new LockingView(sync);

	// this thread serializes and gets stuck there
	final CheckedThread thread = new CheckedThread("serializer") {
		@Override
		public void go() throws Exception {
			serializer.serialize("a value", lockingOut);
		}
	};

	thread.start();
	sync.awaitBlocker();

	// this should fail with an exception
	try {
		serializer.serialize("value", regularOut);
		fail("should have failed with an exception");
	}
	catch (IllegalStateException e) {
		// expected
	}
	finally {
		// release the thread that serializes
		sync.releaseBlocker();
	}

	// this propagates exceptions from the spawned thread
	thread.sync();
}
 
Example #11
Source File: KryoSerializerConcurrencyTest.java    From flink with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}
 
Example #12
Source File: TaskExecutorPartitionLifecycleTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testBlockingLocalPartitionReleaseDoesNotBlockTaskExecutor() throws Exception {
	BlockerSync sync = new BlockerSync();
	ResultPartitionManager blockingResultPartitionManager = new ResultPartitionManager() {
		@Override
		public void releasePartition(ResultPartitionID partitionId, Throwable cause) {
			sync.blockNonInterruptible();
			super.releasePartition(partitionId, cause);
		}
	};

	NettyShuffleEnvironment shuffleEnvironment = new NettyShuffleEnvironmentBuilder()
		.setResultPartitionManager(blockingResultPartitionManager)
		.setIoExecutor(TEST_EXECUTOR_SERVICE_RESOURCE.getExecutor())
		.build();

	final CompletableFuture<ResultPartitionID> startTrackingFuture = new CompletableFuture<>();
	final TaskExecutorPartitionTracker partitionTracker = new TaskExecutorPartitionTrackerImpl(shuffleEnvironment) {
		@Override
		public void startTrackingPartition(JobID producingJobId, TaskExecutorPartitionInfo partitionInfo) {
			super.startTrackingPartition(producingJobId, partitionInfo);
			startTrackingFuture.complete(partitionInfo.getResultPartitionId());
		}
	};

	try {
		internalTestPartitionRelease(
			partitionTracker,
			shuffleEnvironment,
			startTrackingFuture,
			(jobId, resultPartitionDeploymentDescriptor, taskExecutor, taskExecutorGateway) -> {
				final IntermediateDataSetID dataSetId = resultPartitionDeploymentDescriptor.getResultId();

				taskExecutorGateway.releaseClusterPartitions(Collections.singleton(dataSetId), timeout);

				// execute some operation to check whether the TaskExecutor is blocked
				taskExecutorGateway.canBeReleased().get(5, TimeUnit.SECONDS);
			}
		);
	} finally {
		sync.releaseBlocker();
	}
}
 
Example #13
Source File: KryoSerializerConcurrencyTest.java    From flink with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}
 
Example #14
Source File: AvroSerializerConcurrencyTest.java    From flink with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}
 
Example #15
Source File: AvroSerializerConcurrencyTest.java    From flink with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}
 
Example #16
Source File: KryoSerializerConcurrencyTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}
 
Example #17
Source File: AvroSerializerConcurrencyTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
LockingView(BlockerSync blocker) {
	super(32);
	this.blocker = blocker;
}