Java Code Examples for java.util.concurrent.ExecutorService#submit()
The following examples show how to use
java.util.concurrent.ExecutorService#submit() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CopyMonitor.java From ibm-cos-sdk-java with Apache License 2.0 | 6 votes |
/** * Constructs a new watcher for copy operation, and then immediately submits * it to the thread pool. * * @param manager * The {@link TransferManager} that owns this copy request. * @param threadPool * The {@link ExecutorService} to which we should submit new * tasks. * @param multipartCopyCallable * The callable responsible for processing the copy * asynchronously * @param copyObjectRequest * The original CopyObject request */ public static CopyMonitor create( TransferManager manager, CopyImpl transfer, ExecutorService threadPool, CopyCallable multipartCopyCallable, CopyObjectRequest copyObjectRequest, ProgressListenerChain progressListenerChain) { CopyMonitor copyMonitor = new CopyMonitor(manager, transfer, threadPool, multipartCopyCallable, copyObjectRequest, progressListenerChain); Future<CopyResult> thisFuture = threadPool.submit(copyMonitor); // Use an atomic compareAndSet to prevent a possible race between the // setting of the CopyMonitor's futureReference, and setting the // CompleteMultipartCopy's futureReference within the call() method. // We only want to set the futureReference to CopyMonitor's futureReference if the // current value is null, otherwise the futureReference that's set is // CompleteMultipartCopy's which is ultimately what we want. copyMonitor.futureReference.compareAndSet(null, thisFuture); return copyMonitor; }
Example 2
Source File: TransactionalCacheTest.java From mybatis-test with Apache License 2.0 | 6 votes |
@Test public void testTransactional() throws Exception { ExecutorService es = Executors.newFixedThreadPool(2); Future<String> fa = es.submit(this::transactionalA); Future<String> fb = es.submit(this::transactionalB); countDownLatch.countDown(); es.awaitTermination(6, TimeUnit.SECONDS); System.out.println(fa.get()); System.out.println("\n -------- 分割线 ------- \n"); System.out.println(fb.get()); System.out.println(); System.out.println(); System.out.println(); }
Example 3
Source File: AbstractBootstrapServer.java From InChat with Apache License 2.0 | 6 votes |
private void initSsl(InitNetty serverBean){ ExecutorService executorService = Executors.newCachedThreadPool(); executorService.submit(() -> {}); String algorithm = SystemPropertyUtil.get("ssl.KeyManagerFactory.algorithm"); if (algorithm == null) { algorithm = "SunX509"; } SSLContext serverContext; try { // KeyStore ks = KeyStore.getInstance("JKS"); ks.load( SecureSocketSslContextFactory.class.getResourceAsStream(serverBean.getJksFile()), serverBean.getJksStorePassword().toCharArray()); KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm); kmf.init(ks,serverBean.getJksCertificatePassword().toCharArray()); serverContext = SSLContext.getInstance(PROTOCOL); serverContext.init(kmf.getKeyManagers(), null, null); } catch (Exception e) { throw new Error( "Failed to initialize the server-side SSLContext", e); } SERVER_CONTEXT = serverContext; }
Example 4
Source File: TestCAS.java From new-bull with MIT License | 6 votes |
private static void testNonAtomic() throws InterruptedException { final NonAtomicInteger counter = new NonAtomicInteger(); final int threadCount = 10; final int countPerThread = 100000; ExecutorService threadPool = Executors.newFixedThreadPool(threadCount); for (int i = 0; i < threadCount; i++) { threadPool.submit(() -> { for (int j = 0; j < countPerThread; j++) { counter.incr(); } }); } threadPool.awaitTermination(10, TimeUnit.SECONDS); threadPool.shutdown(); System.out.println("expected : " + (threadCount * countPerThread)); System.out.println("actual : " + counter.get()); }
Example 5
Source File: ProxyClassPathImplementationTest.java From netbeans with Apache License 2.0 | 6 votes |
public void testDeadLock() throws Exception{ List<PathResourceImplementation> resources = Collections.<PathResourceImplementation>emptyList(); final ReentrantLock lock = new ReentrantLock (false); final CountDownLatch signal = new CountDownLatch (1); final ClassPath cp = ClassPathFactory.createClassPath(ClassPathSupport.createProxyClassPathImplementation(new ClassPathImplementation[] {new LockClassPathImplementation (resources,lock, signal)})); lock.lock(); final ExecutorService es = Executors.newSingleThreadExecutor(); try { es.submit(new Runnable () { public void run () { cp.entries(); } }); signal.await(); cp.entries(); } finally { es.shutdownNow(); } }
Example 6
Source File: MTEntryPointsTest.java From kogito-runtimes with Apache License 2.0 | 6 votes |
/** * Inserts events using multiple threads into one EntryPoint. The insert * operation is synchronized on corresponding SessionEntryPoint instance. */ @Test public void testOneEntryPoint() throws Exception { final EntryPoint firstThreadEntryPoint = kieSession.getEntryPoint("FirstStream"); final int numInsertersInEachEntryPoint = 10; final ExecutorService executorService = Executors.newFixedThreadPool(numInsertersInEachEntryPoint); try { final List<Future<?>> futures = new ArrayList<>(); for (int i = 0; i < numInsertersInEachEntryPoint; i++) { // future for exception watching final Future<?> futureForFirstThread = executorService.submit( new TestInserter(kieSession, firstThreadEntryPoint)); futures.add(futureForFirstThread); } for (final Future<?> f : futures) { f.get(30, TimeUnit.SECONDS); } } finally { executorService.shutdownNow(); } }
Example 7
Source File: ShareObj.java From banyan with MIT License | 5 votes |
public static void main(String[] args) { ShareCounter shareCounter = new ShareCounter(); ExecutorService executorService = Executors.newFixedThreadPool(4); for (int i = 0; i < 3; i++) executorService.submit(new Task(shareCounter)); try { Thread.currentThread().join(500);//测试使用 System.err.println(shareCounter.counter); } catch (InterruptedException e) { e.printStackTrace(); } executorService.shutdownNow(); }
Example 8
Source File: TestRetryProxy.java From big-c with Apache License 2.0 | 5 votes |
@Test public void testRetryInterruptible() throws Throwable { final UnreliableInterface unreliable = (UnreliableInterface) RetryProxy.create(UnreliableInterface.class, unreliableImpl, retryUpToMaximumTimeWithFixedSleep(10, 10, TimeUnit.SECONDS)); final CountDownLatch latch = new CountDownLatch(1); final AtomicReference<Thread> futureThread = new AtomicReference<Thread>(); ExecutorService exec = Executors.newSingleThreadExecutor(); Future<Throwable> future = exec.submit(new Callable<Throwable>(){ @Override public Throwable call() throws Exception { futureThread.set(Thread.currentThread()); latch.countDown(); try { unreliable.alwaysFailsWithFatalException(); } catch (UndeclaredThrowableException ute) { return ute.getCause(); } return null; } }); latch.await(); Thread.sleep(1000); // time to fail and sleep assertTrue(futureThread.get().isAlive()); futureThread.get().interrupt(); Throwable e = future.get(1, TimeUnit.SECONDS); // should return immediately assertNotNull(e); assertEquals(InterruptedException.class, e.getClass()); assertEquals("sleep interrupted", e.getMessage()); }
Example 9
Source File: ThreadPoolTaskScheduler.java From spring-analysis-note with MIT License | 5 votes |
@Override public Future<?> submit(Runnable task) { ExecutorService executor = getScheduledExecutor(); try { return executor.submit(errorHandlingTask(task, false)); } catch (RejectedExecutionException ex) { throw new TaskRejectedException("Executor [" + executor + "] did not accept task: " + task, ex); } }
Example 10
Source File: FutureThread.java From MediaSDK with Apache License 2.0 | 5 votes |
public FutureThread(final ExecutorService pool, final FutureRunnable<T> runnable) { pool.submit(new Runnable() { @Override public void run() { try { setComplete(runnable.run()); } catch (Exception e) { setComplete(e); } } }); }
Example 11
Source File: ScheduledExecutorTest.java From j2objc with Apache License 2.0 | 5 votes |
/** * completed submit of callable returns result */ public void testSubmitCallable() throws Exception { final ExecutorService e = new ScheduledThreadPoolExecutor(2); try (PoolCleaner cleaner = cleaner(e)) { Future<String> future = e.submit(new StringTask()); String result = future.get(); assertSame(TEST_STRING, result); } }
Example 12
Source File: MultisetSemaphoreTest.java From bazel with Apache License 2.0 | 4 votes |
@Test public void testConcurrentRace_AllSameSizedCombinations() throws Exception { // When we have n values int n = 10; ImmutableSet.Builder<String> valsBuilder = ImmutableSet.builder(); for (int i = 0; i < n; i++) { valsBuilder.add("val-" + i); } ImmutableSet<String> vals = valsBuilder.build(); int k = 5; // And we have all combinations of size k of these n values Set<Set<String>> combinations = Sets.combinations(vals, k); int numCombinations = combinations.size(); // And we have a MultisetSemaphore final MultisetSemaphore<String> multisetSemaphore = MultisetSemaphore.newBuilder() // with K max num unique values, .maxNumUniqueValues(k) .build(); // And a ExecutorService with nCk threads, ExecutorService executorService = Executors.newFixedThreadPool(numCombinations); // And a recorder for thrown exceptions, ThrowableRecordingRunnableWrapper wrapper = new ThrowableRecordingRunnableWrapper("testConcurrentRace_AllSameSizedCombinations"); // And a ConcurrentHashMultiset for counting the multiplicities of the values ourselves, ConcurrentHashMultiset<String> counts = ConcurrentHashMultiset.create(); for (Set<String> combination : combinations) { // And, for each of the nCk combinations, we submit a Runnable, that @SuppressWarnings("unused") Future<?> possiblyIgnoredError = executorService.submit( wrapper.wrap( new Runnable() { @Override public void run() { try { // Tries to acquire permits for its set of k values, multisetSemaphore.acquireAll(combination); // And then verifies that the multiplicities are as expected, combination.forEach(counts::add); assertThat(counts.entrySet().size()).isAtMost(k); combination.forEach(counts::remove); // And then releases the permits. multisetSemaphore.releaseAll(combination); } catch (InterruptedException e) { throw new IllegalStateException(e); } } })); } // Then all of our Runnables completed (without deadlock!), as expected, boolean interrupted = ExecutorUtil.interruptibleShutdown(executorService); // And also none of them threw any Exceptions. assertThat(wrapper.getFirstThrownError()).isNull(); if (interrupted) { Thread.currentThread().interrupt(); throw new InterruptedException(); } }
Example 13
Source File: RecordWriterTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests a fix for FLINK-2089. * * @see <a href="https://issues.apache.org/jira/browse/FLINK-2089">FLINK-2089</a> */ @Test public void testClearBuffersAfterInterruptDuringBlockingBufferRequest() throws Exception { ExecutorService executor = null; try { executor = Executors.newSingleThreadExecutor(); final CountDownLatch sync = new CountDownLatch(2); final TrackingBufferRecycler recycler = new TrackingBufferRecycler(); final MemorySegment memorySegment = MemorySegmentFactory.allocateUnpooledSegment(4); // Return buffer for first request, but block for all following requests. Answer<BufferBuilder> request = new Answer<BufferBuilder>() { @Override public BufferBuilder answer(InvocationOnMock invocation) throws Throwable { sync.countDown(); if (sync.getCount() == 1) { return new BufferBuilder(memorySegment, recycler); } final Object o = new Object(); synchronized (o) { while (true) { o.wait(); } } } }; BufferProvider bufferProvider = mock(BufferProvider.class); when(bufferProvider.requestBufferBuilderBlocking()).thenAnswer(request); ResultPartitionWriter partitionWriter = new RecyclingPartitionWriter(bufferProvider); final RecordWriter<IntValue> recordWriter = new RecordWriter<IntValue>(partitionWriter); Future<?> result = executor.submit(new Callable<Void>() { @Override public Void call() throws Exception { IntValue val = new IntValue(0); try { recordWriter.emit(val); recordWriter.flushAll(); recordWriter.emit(val); } catch (InterruptedException e) { recordWriter.clearBuffers(); } return null; } }); sync.await(); // Interrupt the Thread. // // The second emit call requests a new buffer and blocks the thread. // When interrupting the thread at this point, clearing the buffers // should not recycle any buffer. result.cancel(true); recordWriter.clearBuffers(); // Verify that buffer have been requested twice verify(bufferProvider, times(2)).requestBufferBuilderBlocking(); // Verify that the written out buffer has only been recycled once // (by the partition writer). assertEquals(1, recycler.getRecycledMemorySegments().size()); assertEquals(memorySegment, recycler.getRecycledMemorySegments().get(0)); } finally { if (executor != null) { executor.shutdown(); } } }
Example 14
Source File: JCudaDriverStreamCallbacks.java From jcuda-samples with MIT License | 4 votes |
/** * Create a Workload instance. This method is called by multiple host * threads, to create the individual workloads, and to send the * commands for processing the workloads to CUDA * * @param index The index of the workload * @param executor The executor service */ private static void createWorkloadOnHost( final int index, final ExecutorService executor) { // Make sure that the CUDA context is current for the calling thread cuCtxSetCurrent(context); // Initialize the workload, and create the CUDA stream System.out.println(index + ": Initializing workload"); final Workload workload = new Workload(); workload.index = index; workload.stream = new CUstream(); cuStreamCreate(workload.stream, 0); // Create the host data of the workload System.out.println(index + ": Create host data"); workload.hostData = new Pointer(); cuMemHostAlloc(workload.hostData, WORKLOAD_SIZE * Sizeof.INT, 0); ByteBuffer hostByteBuffer = workload.hostData.getByteBuffer(0, WORKLOAD_SIZE * Sizeof.INT); IntBuffer hostIntBuffer = hostByteBuffer.order(ByteOrder.nativeOrder()).asIntBuffer(); for (int i = 0; i < WORKLOAD_SIZE; i++) { hostIntBuffer.put(i, i); } workload.deviceData = new CUdeviceptr(); cuMemAlloc(workload.deviceData, WORKLOAD_SIZE * Sizeof.INT); // Execute the CUDA commands: // - Copy the host data to the device // - Execute the kernel // - Copy the modified device data back to the host // All this is done asynchronously System.out.println(index + ": Execute CUDA commands"); cuMemcpyHtoDAsync(workload.deviceData, workload.hostData, WORKLOAD_SIZE * Sizeof.INT, workload.stream); Pointer kernelParameters = Pointer.to( Pointer.to(new int[]{WORKLOAD_SIZE}), Pointer.to(workload.deviceData) ); int blockSizeX = 256; int gridSizeX = (WORKLOAD_SIZE + blockSizeX - 1) / blockSizeX; cuLaunchKernel(function, gridSizeX, 1, 1, blockSizeX, 1, 1, 0, workload.stream, kernelParameters, null); cuMemcpyDtoHAsync(workload.hostData, workload.deviceData, WORKLOAD_SIZE * Sizeof.INT, workload.stream); // Define the callback that will be called when all CUDA commands // on the stream have finished. This callback will forward the // workload to the "finishWorkloadOnHost" method. CUstreamCallback callback = new CUstreamCallback() { @Override public void call( CUstream hStream, int status, final Object userData) { System.out.println(index + ": Callback was called"); Runnable runnable = new Runnable() { @Override public void run() { finishWorkloadOnHost(userData); } }; executor.submit(runnable); } }; cuStreamAddCallback(workload.stream, callback, workload, 0); }
Example 15
Source File: AdvancedGeolocation.java From cordova-plugin-advanced-geolocation with Apache License 2.0 | 4 votes |
private void startLocation(){ // Misc. note: If you see the message "Attempted to send a second callback for ID:" then you need // to make sure to set pluginResult.setKeepCallback(true); // We want to prevent multiple instances of controllers from running! if(_gpsController != null || _networkLocationController != null || _cellLocationController != null){ stopLocation(); } final boolean networkEnabled = isInternetConnected(_cordovaActivity.getApplicationContext()); ExecutorService threadPool = cordova.getThreadPool(); if(_providers.equalsIgnoreCase(PROVIDERS_ALL)){ _gpsController = new GPSController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize); _gpsFuture = threadPool.submit(_gpsController); _networkLocationController = new NetworkLocationController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize); _networkFuture = threadPool.submit(_networkLocationController); // Reference: https://developer.android.com/reference/android/telephony/TelephonyManager.html#getAllCellInfo() // Reference: https://developer.android.com/reference/android/telephony/CellIdentityWcdma.html (added at API 18) if (Build.VERSION.SDK_INT < MIN_API_LEVEL){ cellDataNotAllowed(); } else { _cellLocationController = new CellLocationController(networkEnabled, _signalStrength, _cordova,_callbackContext); _cellularFuture = threadPool.submit(_cellLocationController); } } if(_providers.equalsIgnoreCase(PROVIDERS_SOME)){ _gpsController = new GPSController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize); _gpsFuture = threadPool.submit(_gpsController); _networkLocationController = new NetworkLocationController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize); _networkFuture = threadPool.submit(_networkLocationController); } if(_providers.equalsIgnoreCase(PROVIDERS_GPS)){ _gpsController = new GPSController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize); _gpsFuture = threadPool.submit(_gpsController); } if(_providers.equalsIgnoreCase(PROVIDERS_NETWORK)){ _networkLocationController = new NetworkLocationController( _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize); _networkFuture = threadPool.submit(_networkLocationController); } if(_providers.equalsIgnoreCase(PROVIDERS_CELL)){ // Reference: https://developer.android.com/reference/android/telephony/TelephonyManager.html#getAllCellInfo() // Reference: https://developer.android.com/reference/android/telephony/CellIdentityWcdma.html if (Build.VERSION.SDK_INT < MIN_API_LEVEL){ cellDataNotAllowed(); } else { _cellLocationController = new CellLocationController(networkEnabled,_signalStrength ,_cordova,_callbackContext); _cellularFuture = threadPool.submit(_cellLocationController); } } }
Example 16
Source File: KieModuleRepoTest.java From kogito-runtimes with Apache License 2.0 | 4 votes |
/** * TESTS ---------------------------------------------------------------------------------------------------------------------- */ // simultaneous requests to deploy two new deployments (different versions) // for an empty/new GA artifactMap @Test public void testDeployTwoArtifactVersionsSameTime() throws Exception { final String groupId = "org"; final String artifactId = "one"; final String firstVersion = "1.0"; final String secondVersion = "1.0-NEW-FEATURE"; final CyclicBarrier storeOperationBarrier = new CyclicBarrier(2); final CyclicBarrier threadsFinishedBarrier = new CyclicBarrier(3); final Thread firstThread = new Thread( getStoreArtifactRunnable(kieModuleRepo, groupId, artifactId, firstVersion, storeOperationBarrier, threadsFinishedBarrier)); final Thread secondThread = new Thread( getStoreArtifactRunnable(kieModuleRepo, groupId, artifactId, secondVersion, storeOperationBarrier, threadsFinishedBarrier)); final ExecutorService executor = Executors.newFixedThreadPool(2); try { firstThread.setName("normal"); executor.submit(firstThread); secondThread.setName("newFeature"); executor.submit(secondThread); waitFor(threadsFinishedBarrier); } finally { executor.shutdownNow(); } final String ga = groupId + ":" + artifactId; final Map<ComparableVersion, KieModule> artifactMap = kieModuleRepo.kieModules.get(ga); final ComparableVersion normalVersion = new ComparableVersion(firstVersion); final KieModule normalKieModule = artifactMap.get(normalVersion); final ComparableVersion newFeatureVersion = new ComparableVersion(secondVersion); final KieModule newFeatureKieModule = artifactMap.get(newFeatureVersion); assertNotNull(normalKieModule, "Race condition occurred: normal KieModule disappeared from KieModuleRepo!"); assertNotNull(newFeatureKieModule, "Race condition occurred: new feature KieModule disappeared from KieModuleRepo!"); }
Example 17
Source File: LocalSpawnRunnerTest.java From bazel with Apache License 2.0 | 4 votes |
@Test public void interruptWaitsForProcessExit() throws Exception { assumeTrue(OS.getCurrent() != OS.WINDOWS); File tempDirFile = TestUtils.makeTempDir(); tempDirFile.deleteOnExit(); FileSystem fs = new JavaIoFileSystem(DigestHashFunction.getDefaultUnchecked()); Path tempDir = fs.getPath(tempDirFile.getPath()); LocalSpawnRunner runner = new LocalSpawnRunner( tempDir, Options.getDefaults(LocalExecutionOptions.class), resourceManager, LocalEnvProvider.forCurrentOs(ImmutableMap.of()), /*binTools=*/ null, /*processWrapper=*/ null, Mockito.mock(RunfilesTreeUpdater.class)); FileOutErr fileOutErr = new FileOutErr(tempDir.getRelative("stdout"), tempDir.getRelative("stderr")); SpawnExecutionContextForTesting policy = new SpawnExecutionContextForTesting(fileOutErr); // This test to exercise a race condition by attempting an operation multiple times. We can get // false positives (the test passing without us catching a problem), so try a few times. When // implementing this fix on 2019-09-11, this specific configuration was sufficient to catch the // previously-existent bug. int tries = 10; int delaySeconds = 1; Path content = tempDir.getChild("content"); Path started = tempDir.getChild("started"); // Start a subprocess that blocks until it is killed, and when it is, writes some output to // a temporary file after some delay. String script = "trap 'sleep " + delaySeconds + "; echo foo >" + content.getPathString() + "; exit 1' TERM; " + "touch " + started.getPathString() + "; " + "while :; do " + " echo 'waiting to be killed'; " + " sleep 1; " + "done"; Spawn spawn = new SpawnBuilder("/bin/sh", "-c", script).build(); ExecutorService executor = Executors.newSingleThreadExecutor(); try { for (int i = 0; i < tries; i++) { content.delete(); started.delete(); Semaphore interruptCaught = new Semaphore(0); Future<?> future = executor.submit( () -> { try { runner.exec(spawn, policy); } catch (InterruptedException e) { interruptCaught.release(); } catch (Throwable t) { throw new IllegalStateException(t); } }); // Wait until we know the subprocess has started so that delivering a termination signal // to it triggers the delayed write to the file. while (!started.exists()) { Thread.sleep(1); } future.cancel(true); interruptCaught.acquireUninterruptibly(); // At this point, the subprocess must have fully stopped so write some content to the file // and expect that these contents remain unmodified. FileSystemUtils.writeContent(content, StandardCharsets.UTF_8, "bar"); // Wait for longer than the spawn takes to exit before we check the file contents to ensure // that we properly awaited for termination of the subprocess. Thread.sleep(delaySeconds * 2 * 1000); assertThat(FileSystemUtils.readContent(content, StandardCharsets.UTF_8)).isEqualTo("bar"); } } finally { executor.shutdown(); } }
Example 18
Source File: SGDNetworkTrainer.java From JSAT with GNU General Public License v3.0 | 4 votes |
/** * Applies dropout to the given matrix * @param X the matrix to dropout values from * @param randThresh the threshold that a random integer must be less than to get dropped out * @param rand the source of randomness * @param ex the source of threads for parlallel computation, or {@code null} */ private static void applyDropout(final Matrix X, final int randThresh, final Random rand, ExecutorService ex) { if (ex == null) { for (int i = 0; i < X.rows(); i++) for (int j = 0; j < X.cols(); j++) if (rand.nextInt() < randThresh) X.set(i, j, 0.0); } else { final CountDownLatch latch = new CountDownLatch(SystemInfo.LogicalCores); for(int id = 0; id < SystemInfo.LogicalCores; id++) { final int ID = id; ex.submit(new Runnable() { @Override public void run() { for (int i = ID; i < X.rows(); i+=SystemInfo.LogicalCores) for (int j = 0; j < X.cols(); j++) if (rand.nextInt() < randThresh) X.set(i, j, 0.0); latch.countDown(); } }); } try { latch.await(); } catch (InterruptedException ex1) { Logger.getLogger(SGDNetworkTrainer.class.getName()).log(Level.SEVERE, null, ex1); } } }
Example 19
Source File: ResultPartitionTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testInitializeMoreStateThanBuffer() throws Exception { final int totalBuffers = 2; // the total buffers are less than the requirement from total states final int totalStates = 5; final int[] states = {1, 2, 3, 4}; final int bufferSize = states.length * Integer.BYTES; final NetworkBufferPool globalPool = new NetworkBufferPool(totalBuffers, bufferSize, 1); final ChannelStateReader stateReader = new FiniteChannelStateReader(totalStates, states); final ResultPartition partition = new ResultPartitionBuilder() .setNetworkBufferPool(globalPool) .build(); final ExecutorService executor = Executors.newFixedThreadPool(1); try { final Callable<Void> partitionConsumeTask = () -> { for (ResultSubpartition subpartition : partition.getAllPartitions()) { final ResultSubpartitionView view = new PipelinedSubpartitionView( (PipelinedSubpartition) subpartition, new NoOpBufferAvailablityListener()); int numConsumedBuffers = 0; while (numConsumedBuffers != totalStates) { ResultSubpartition.BufferAndBacklog bufferAndBacklog = view.getNextBuffer(); if (bufferAndBacklog != null) { Buffer buffer = bufferAndBacklog.buffer(); BufferBuilderAndConsumerTest.assertContent( buffer, partition.getBufferPool() .getSubpartitionBufferRecyclers()[subpartition.getSubPartitionIndex()], states); buffer.recycleBuffer(); numConsumedBuffers++; } else { Thread.sleep(5); } } assertNull(view.getNextBuffer()); } return null; }; Future<Void> result = executor.submit(partitionConsumeTask); partition.setup(); partition.readRecoveredState(stateReader); // wait the partition consume task finish result.get(20, TimeUnit.SECONDS); // destroy the local pool to verify that all the requested buffers by partition are recycled partition.getBufferPool().lazyDestroy(); assertEquals(totalBuffers, globalPool.getNumberOfAvailableMemorySegments()); } finally { // cleanup executor.shutdown(); globalPool.destroyAllBufferPools(); globalPool.destroy(); } }
Example 20
Source File: SpillableSubpartitionTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Tests a fix for FLINK-12544. * * @see <a href="https://issues.apache.org/jira/browse/FLINK-12544">FLINK-12544</a> */ @Test public void testConcurrentRequestAndReleaseMemory() throws Exception { final ExecutorService executor = Executors.newFixedThreadPool(2); final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32); try { final CountDownLatch blockLatch = new CountDownLatch(1); final CountDownLatch doneLatch = new CountDownLatch(1); final IOManager ioManager = new IOManagerAsyncWithCountDownLatch(blockLatch, doneLatch); final ResultPartitionWithCountDownLatch partition = new ResultPartitionWithCountDownLatch( "Test", new NoOpTaskActions(), new JobID(), new ResultPartitionID(), ResultPartitionType.BLOCKING, 1, 1, new ResultPartitionManager(), new NoOpResultPartitionConsumableNotifier(), ioManager, true, doneLatch, blockLatch); final BufferPool bufferPool = networkBufferPool.createBufferPool(1, 1, Optional.of(partition)); partition.registerBufferPool(bufferPool); final BufferBuilder firstBuffer = bufferPool.requestBufferBuilderBlocking(); partition.addBufferConsumer(firstBuffer.createBufferConsumer(), 0); // Finishes the buffer consumer which could be recycled during SpillableSubpartition#releaseMemory firstBuffer.finish(); Future<Void> future = executor.submit(new Callable<Void>() { @Override public Void call() throws Exception { //Occupies the lock in SpillableSubpartition#releaseMemory, trying to get the lock in LocalBufferPool#recycle partition.releaseMemory(1); return null; } }); final CompletableFuture<?> firstCallFuture = partition.getFirstCallFuture(); firstCallFuture.thenRunAsync(() -> { try { // There are no available buffers in pool, so trigger release memory in SpillableSubpartition. // Occupies the lock in LocalBufferPool, and trying to get the lock in SpillableSubpartition. BufferBuilder secondBuffer = bufferPool.requestBufferBuilderBlocking(); assertThat(firstBuffer, is(equalTo(secondBuffer))); } catch (IOException | InterruptedException ex) { fail("Should not throw any exceptions!"); } }, executor); future.get(); } finally { networkBufferPool.destroyAllBufferPools(); networkBufferPool.destroy(); executor.shutdown(); } }