Java Code Examples for java.util.concurrent.ExecutorService#submit()

/**
 * Constructs a new watcher for copy operation, and then immediately submits
 * it to the thread pool.
 *
 * @param manager
 *            The {@link TransferManager} that owns this copy request.
 * @param threadPool
 *            The {@link ExecutorService} to which we should submit new
 *            tasks.
 * @param multipartCopyCallable
 *            The callable responsible for processing the copy
 *            asynchronously
 * @param copyObjectRequest
 *            The original CopyObject request
 */
public static CopyMonitor create(
        TransferManager manager,
        CopyImpl transfer,
        ExecutorService threadPool,
        CopyCallable multipartCopyCallable,
        CopyObjectRequest copyObjectRequest,
        ProgressListenerChain progressListenerChain) {

    CopyMonitor copyMonitor = new CopyMonitor(manager, transfer,
            threadPool, multipartCopyCallable, copyObjectRequest,
            progressListenerChain);
    Future<CopyResult> thisFuture = threadPool.submit(copyMonitor);
    // Use an atomic compareAndSet to prevent a possible race between the
    // setting of the CopyMonitor's futureReference, and setting the
    // CompleteMultipartCopy's futureReference within the call() method.
    // We only want to set the futureReference to CopyMonitor's futureReference if the
    // current value is null, otherwise the futureReference that's set is
    // CompleteMultipartCopy's which is ultimately what we want.
    copyMonitor.futureReference.compareAndSet(null, thisFuture);
    return copyMonitor;
}
 

@Test
public void testTransactional() throws Exception {
    ExecutorService es = Executors.newFixedThreadPool(2);
    Future<String> fa = es.submit(this::transactionalA);
    Future<String> fb = es.submit(this::transactionalB);

    countDownLatch.countDown();
    es.awaitTermination(6, TimeUnit.SECONDS);

    System.out.println(fa.get());
    System.out.println("\n -------- 分割线 ------- \n");
    System.out.println(fb.get());

    System.out.println();
    System.out.println();
    System.out.println();
}
 

private void initSsl(InitNetty serverBean){
    ExecutorService executorService = Executors.newCachedThreadPool();
    executorService.submit(() -> {});
    String algorithm = SystemPropertyUtil.get("ssl.KeyManagerFactory.algorithm");
    if (algorithm == null) {
        algorithm = "SunX509";
    }
    SSLContext serverContext;
    try {
        //
        KeyStore ks = KeyStore.getInstance("JKS");
        ks.load( SecureSocketSslContextFactory.class.getResourceAsStream(serverBean.getJksFile()),
                serverBean.getJksStorePassword().toCharArray());
        KeyManagerFactory kmf = KeyManagerFactory.getInstance(algorithm);
        kmf.init(ks,serverBean.getJksCertificatePassword().toCharArray());
        serverContext = SSLContext.getInstance(PROTOCOL);
        serverContext.init(kmf.getKeyManagers(), null, null);
    } catch (Exception e) {
        throw new Error(
                "Failed to initialize the server-side SSLContext", e);
    }
    SERVER_CONTEXT = serverContext;
}
 

private static void testNonAtomic() throws InterruptedException {

        final NonAtomicInteger counter = new NonAtomicInteger();
        final int threadCount = 10;
        final int countPerThread = 100000;

        ExecutorService threadPool = Executors.newFixedThreadPool(threadCount);
        for (int i = 0; i < threadCount; i++) {
            threadPool.submit(() -> {
                for (int j = 0; j < countPerThread; j++) {
                    counter.incr();
                }
            });
        }
        threadPool.awaitTermination(10, TimeUnit.SECONDS);
        threadPool.shutdown();

        System.out.println("expected : " + (threadCount * countPerThread));
        System.out.println("actual : " + counter.get());

    }
 

public void testDeadLock() throws Exception{
    List<PathResourceImplementation> resources = Collections.<PathResourceImplementation>emptyList();
    final ReentrantLock lock = new ReentrantLock (false);
    final CountDownLatch signal = new CountDownLatch (1);
    final ClassPath cp = ClassPathFactory.createClassPath(ClassPathSupport.createProxyClassPathImplementation(new ClassPathImplementation[] {new LockClassPathImplementation (resources,lock, signal)}));
    lock.lock();
    final ExecutorService es = Executors.newSingleThreadExecutor();        
    try {
        es.submit(new Runnable () {
            public void run () {
                cp.entries();
            }
        });
        signal.await();
        cp.entries();
    } finally {
        es.shutdownNow();
    }
}
 

/**
 * Inserts events using multiple threads into one EntryPoint. The insert
 * operation is synchronized on corresponding SessionEntryPoint instance.
 */
@Test
public void testOneEntryPoint() throws Exception {
    final EntryPoint firstThreadEntryPoint = kieSession.getEntryPoint("FirstStream");

    final int numInsertersInEachEntryPoint = 10;
    final ExecutorService executorService = Executors.newFixedThreadPool(numInsertersInEachEntryPoint);
    try {
        final List<Future<?>> futures = new ArrayList<>();

        for (int i = 0; i < numInsertersInEachEntryPoint; i++) {
            // future for exception watching
            final Future<?> futureForFirstThread = executorService.submit(
                    new TestInserter(kieSession, firstThreadEntryPoint));
            futures.add(futureForFirstThread);
        }

        for (final Future<?> f : futures) {
            f.get(30, TimeUnit.SECONDS);
        }
    } finally {
        executorService.shutdownNow();
    }
}
 

public static void main(String[] args) {
    ShareCounter shareCounter = new ShareCounter();
    ExecutorService executorService = Executors.newFixedThreadPool(4);

    for (int i = 0; i < 3; i++)
        executorService.submit(new Task(shareCounter));

    try {
        Thread.currentThread().join(500);//测试使用
        System.err.println(shareCounter.counter);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    executorService.shutdownNow();
}
 

@Test
public void testRetryInterruptible() throws Throwable {
  final UnreliableInterface unreliable = (UnreliableInterface)
      RetryProxy.create(UnreliableInterface.class, unreliableImpl,
          retryUpToMaximumTimeWithFixedSleep(10, 10, TimeUnit.SECONDS));
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicReference<Thread> futureThread = new AtomicReference<Thread>();
  ExecutorService exec = Executors.newSingleThreadExecutor();
  Future<Throwable> future = exec.submit(new Callable<Throwable>(){
    @Override
    public Throwable call() throws Exception {
      futureThread.set(Thread.currentThread());
      latch.countDown();
      try {
        unreliable.alwaysFailsWithFatalException();
      } catch (UndeclaredThrowableException ute) {
        return ute.getCause();
      }
      return null;
    }
  });
  latch.await();
  Thread.sleep(1000); // time to fail and sleep
  assertTrue(futureThread.get().isAlive());
  futureThread.get().interrupt();
  Throwable e = future.get(1, TimeUnit.SECONDS); // should return immediately 
  assertNotNull(e);
  assertEquals(InterruptedException.class, e.getClass());
  assertEquals("sleep interrupted", e.getMessage());
}
 

@Override
public Future<?> submit(Runnable task) {
	ExecutorService executor = getScheduledExecutor();
	try {
		return executor.submit(errorHandlingTask(task, false));
	}
	catch (RejectedExecutionException ex) {
		throw new TaskRejectedException("Executor [" + executor + "] did not accept task: " + task, ex);
	}
}
 

public FutureThread(final ExecutorService pool, final FutureRunnable<T> runnable) {
    pool.submit(new Runnable() {
        @Override
        public void run() {
            try {
                setComplete(runnable.run());
            }
            catch (Exception e) {
                setComplete(e);
            }
        }
    });
}
 

/**
 * completed submit of callable returns result
 */
public void testSubmitCallable() throws Exception {
    final ExecutorService e = new ScheduledThreadPoolExecutor(2);
    try (PoolCleaner cleaner = cleaner(e)) {
        Future<String> future = e.submit(new StringTask());
        String result = future.get();
        assertSame(TEST_STRING, result);
    }
}
 

@Test
public void testConcurrentRace_AllSameSizedCombinations() throws Exception {
  // When we have n values
  int n = 10;
  ImmutableSet.Builder<String> valsBuilder = ImmutableSet.builder();
  for (int i = 0; i < n; i++) {
    valsBuilder.add("val-" + i);
  }
  ImmutableSet<String> vals = valsBuilder.build();
  int k = 5;
  // And we have all combinations of size k of these n values
  Set<Set<String>> combinations = Sets.combinations(vals, k);
  int numCombinations = combinations.size();
  // And we have a MultisetSemaphore
  final MultisetSemaphore<String> multisetSemaphore = MultisetSemaphore.newBuilder()
      // with K max num unique values,
      .maxNumUniqueValues(k)
      .build();
  // And a ExecutorService with nCk threads,
  ExecutorService executorService = Executors.newFixedThreadPool(numCombinations);
  // And a recorder for thrown exceptions,
  ThrowableRecordingRunnableWrapper wrapper =
      new ThrowableRecordingRunnableWrapper("testConcurrentRace_AllSameSizedCombinations");
  // And a ConcurrentHashMultiset for counting the multiplicities of the values ourselves,
  ConcurrentHashMultiset<String> counts = ConcurrentHashMultiset.create();
  for (Set<String> combination : combinations) {
    // And, for each of the nCk combinations, we submit a Runnable, that
    @SuppressWarnings("unused")
    Future<?> possiblyIgnoredError =
        executorService.submit(
            wrapper.wrap(
                new Runnable() {
                  @Override
                  public void run() {
                    try {
                      // Tries to acquire permits for its set of k values,
                      multisetSemaphore.acquireAll(combination);
                      // And then verifies that the multiplicities are as expected,
                      combination.forEach(counts::add);
                      assertThat(counts.entrySet().size()).isAtMost(k);
                      combination.forEach(counts::remove);
                      // And then releases the permits.
                      multisetSemaphore.releaseAll(combination);
                    } catch (InterruptedException e) {
                      throw new IllegalStateException(e);
                    }
                  }
                }));
  }
  // Then all of our Runnables completed (without deadlock!), as expected,
  boolean interrupted = ExecutorUtil.interruptibleShutdown(executorService);
  // And also none of them threw any Exceptions.
  assertThat(wrapper.getFirstThrownError()).isNull();
  if (interrupted) {
    Thread.currentThread().interrupt();
    throw new InterruptedException();
  }
}
 

/**
 * Tests a fix for FLINK-2089.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-2089">FLINK-2089</a>
 */
@Test
public void testClearBuffersAfterInterruptDuringBlockingBufferRequest() throws Exception {
	ExecutorService executor = null;

	try {
		executor = Executors.newSingleThreadExecutor();

		final CountDownLatch sync = new CountDownLatch(2);

		final TrackingBufferRecycler recycler = new TrackingBufferRecycler();

		final MemorySegment memorySegment = MemorySegmentFactory.allocateUnpooledSegment(4);

		// Return buffer for first request, but block for all following requests.
		Answer<BufferBuilder> request = new Answer<BufferBuilder>() {
			@Override
			public BufferBuilder answer(InvocationOnMock invocation) throws Throwable {
				sync.countDown();

				if (sync.getCount() == 1) {
					return new BufferBuilder(memorySegment, recycler);
				}

				final Object o = new Object();
				synchronized (o) {
					while (true) {
						o.wait();
					}
				}
			}
		};

		BufferProvider bufferProvider = mock(BufferProvider.class);
		when(bufferProvider.requestBufferBuilderBlocking()).thenAnswer(request);

		ResultPartitionWriter partitionWriter = new RecyclingPartitionWriter(bufferProvider);

		final RecordWriter<IntValue> recordWriter = new RecordWriter<IntValue>(partitionWriter);

		Future<?> result = executor.submit(new Callable<Void>() {
			@Override
			public Void call() throws Exception {
				IntValue val = new IntValue(0);

				try {
					recordWriter.emit(val);
					recordWriter.flushAll();

					recordWriter.emit(val);
				}
				catch (InterruptedException e) {
					recordWriter.clearBuffers();
				}

				return null;
			}
		});

		sync.await();

		// Interrupt the Thread.
		//
		// The second emit call requests a new buffer and blocks the thread.
		// When interrupting the thread at this point, clearing the buffers
		// should not recycle any buffer.
		result.cancel(true);

		recordWriter.clearBuffers();

		// Verify that buffer have been requested twice
		verify(bufferProvider, times(2)).requestBufferBuilderBlocking();

		// Verify that the written out buffer has only been recycled once
		// (by the partition writer).
		assertEquals(1, recycler.getRecycledMemorySegments().size());
		assertEquals(memorySegment, recycler.getRecycledMemorySegments().get(0));
	}
	finally {
		if (executor != null) {
			executor.shutdown();
		}
	}
}
 

/**
 * Create a Workload instance. This method is called by multiple host
 * threads, to create the individual workloads, and to send the 
 * commands for processing the workloads to CUDA
 * 
 * @param index The index of the workload 
 * @param executor The executor service 
 */
private static void createWorkloadOnHost(
    final int index, final ExecutorService executor)
{
    // Make sure that the CUDA context is current for the calling thread
    cuCtxSetCurrent(context);

    // Initialize the workload, and create the CUDA stream

    System.out.println(index + ": Initializing workload");
    final Workload workload = new Workload();
    workload.index = index;
    workload.stream = new CUstream();
    cuStreamCreate(workload.stream, 0);
    
    
    // Create the host data of the workload
    
    System.out.println(index + ": Create host data");
    workload.hostData = new Pointer();
    cuMemHostAlloc(workload.hostData, WORKLOAD_SIZE * Sizeof.INT, 0);
    ByteBuffer hostByteBuffer =
        workload.hostData.getByteBuffer(0, WORKLOAD_SIZE * Sizeof.INT);
    IntBuffer hostIntBuffer = 
        hostByteBuffer.order(ByteOrder.nativeOrder()).asIntBuffer();
    for (int i = 0; i < WORKLOAD_SIZE; i++)
    {
        hostIntBuffer.put(i, i);
    }
    workload.deviceData = new CUdeviceptr();
    cuMemAlloc(workload.deviceData, WORKLOAD_SIZE * Sizeof.INT);

    
    // Execute the CUDA commands:
    // - Copy the host data to the device
    // - Execute the kernel
    // - Copy the modified device data back to the host
    // All this is done asynchronously

    System.out.println(index + ": Execute CUDA commands");

    cuMemcpyHtoDAsync(workload.deviceData, workload.hostData,
        WORKLOAD_SIZE * Sizeof.INT, workload.stream);

    Pointer kernelParameters = Pointer.to(
        Pointer.to(new int[]{WORKLOAD_SIZE}),
        Pointer.to(workload.deviceData)
    );
    int blockSizeX = 256;
    int gridSizeX = (WORKLOAD_SIZE + blockSizeX - 1) / blockSizeX;
    cuLaunchKernel(function, gridSizeX,  1, 1, blockSizeX, 1, 1,
        0, workload.stream, kernelParameters, null);
    
    cuMemcpyDtoHAsync(workload.hostData, workload.deviceData,
        WORKLOAD_SIZE * Sizeof.INT, workload.stream);
    
    
    // Define the callback that will be called when all CUDA commands
    // on the stream have finished. This callback will forward the
    // workload to the "finishWorkloadOnHost" method.
    CUstreamCallback callback = new CUstreamCallback()
    {
        @Override
        public void call(
            CUstream hStream, int status, final Object userData)
        {
            System.out.println(index + ": Callback was called");
            Runnable runnable = new Runnable()
            {
                @Override
                public void run()
                {
                    finishWorkloadOnHost(userData);
                }
            };
            executor.submit(runnable);
        }
    };
    cuStreamAddCallback(workload.stream, callback, workload, 0);
}
 

private void startLocation(){

        // Misc. note: If you see the message "Attempted to send a second callback for ID:" then you need
        // to make sure to set pluginResult.setKeepCallback(true);

        // We want to prevent multiple instances of controllers from running!
        if(_gpsController != null || _networkLocationController != null || _cellLocationController != null){
            stopLocation();
        }

        final boolean networkEnabled = isInternetConnected(_cordovaActivity.getApplicationContext());
        ExecutorService threadPool = cordova.getThreadPool();

        if(_providers.equalsIgnoreCase(PROVIDERS_ALL)){
            _gpsController = new GPSController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize);
            _gpsFuture = threadPool.submit(_gpsController);

            _networkLocationController = new NetworkLocationController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize);
            _networkFuture = threadPool.submit(_networkLocationController);

            // Reference: https://developer.android.com/reference/android/telephony/TelephonyManager.html#getAllCellInfo()
            // Reference: https://developer.android.com/reference/android/telephony/CellIdentityWcdma.html (added at API 18)
            if (Build.VERSION.SDK_INT < MIN_API_LEVEL){
                cellDataNotAllowed();
            }
            else {
                _cellLocationController = new CellLocationController(networkEnabled, _signalStrength, _cordova,_callbackContext);
                _cellularFuture = threadPool.submit(_cellLocationController);
            }
        }
        if(_providers.equalsIgnoreCase(PROVIDERS_SOME)){
            _gpsController = new GPSController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize);
            _gpsFuture = threadPool.submit(_gpsController);

            _networkLocationController = new NetworkLocationController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize);
            _networkFuture = threadPool.submit(_networkLocationController);

        }
        if(_providers.equalsIgnoreCase(PROVIDERS_GPS)){
            _gpsController = new GPSController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _returnSatelliteData, _buffer, _bufferSize);
            _gpsFuture = threadPool.submit(_gpsController);
        }
        if(_providers.equalsIgnoreCase(PROVIDERS_NETWORK)){
            _networkLocationController = new NetworkLocationController(
                    _cordova, _callbackContext, _minDistance, _minTime, _useCache, _buffer, _bufferSize);
            _networkFuture = threadPool.submit(_networkLocationController);
        }
        if(_providers.equalsIgnoreCase(PROVIDERS_CELL)){

            // Reference: https://developer.android.com/reference/android/telephony/TelephonyManager.html#getAllCellInfo()
            // Reference: https://developer.android.com/reference/android/telephony/CellIdentityWcdma.html
            if (Build.VERSION.SDK_INT < MIN_API_LEVEL){
                cellDataNotAllowed();
            }
            else {
                _cellLocationController = new CellLocationController(networkEnabled,_signalStrength ,_cordova,_callbackContext);
                _cellularFuture = threadPool.submit(_cellLocationController);
            }
        }
    }
 

/**
 * TESTS ----------------------------------------------------------------------------------------------------------------------
 */

// simultaneous requests to deploy two new deployments (different versions)
// for an empty/new GA artifactMap
@Test
public void testDeployTwoArtifactVersionsSameTime() throws Exception {
    final String groupId = "org";
    final String artifactId = "one";
    final String firstVersion = "1.0";
    final String secondVersion = "1.0-NEW-FEATURE";

    final CyclicBarrier storeOperationBarrier = new CyclicBarrier(2);
    final CyclicBarrier threadsFinishedBarrier = new CyclicBarrier(3);

    final Thread firstThread = new Thread(
            getStoreArtifactRunnable(kieModuleRepo, groupId, artifactId, firstVersion,
                    storeOperationBarrier, threadsFinishedBarrier));

    final Thread secondThread = new Thread(
            getStoreArtifactRunnable(kieModuleRepo, groupId, artifactId, secondVersion,
                    storeOperationBarrier, threadsFinishedBarrier));

    final ExecutorService executor = Executors.newFixedThreadPool(2);
    try {
        firstThread.setName("normal");
        executor.submit(firstThread);
        secondThread.setName("newFeature");
        executor.submit(secondThread);
        waitFor(threadsFinishedBarrier);
    } finally {
        executor.shutdownNow();
    }

    final String ga = groupId + ":" + artifactId;
    final Map<ComparableVersion, KieModule> artifactMap = kieModuleRepo.kieModules.get(ga);

    final ComparableVersion normalVersion = new ComparableVersion(firstVersion);
    final KieModule normalKieModule = artifactMap.get(normalVersion);
    final ComparableVersion newFeatureVersion = new ComparableVersion(secondVersion);
    final KieModule newFeatureKieModule = artifactMap.get(newFeatureVersion);

    assertNotNull(normalKieModule,
                             "Race condition occurred: normal KieModule disappeared from KieModuleRepo!");
    assertNotNull(newFeatureKieModule,
                             "Race condition occurred: new feature KieModule disappeared from KieModuleRepo!");
}
 

@Test
public void interruptWaitsForProcessExit() throws Exception {
  assumeTrue(OS.getCurrent() != OS.WINDOWS);

  File tempDirFile = TestUtils.makeTempDir();
  tempDirFile.deleteOnExit();
  FileSystem fs = new JavaIoFileSystem(DigestHashFunction.getDefaultUnchecked());
  Path tempDir = fs.getPath(tempDirFile.getPath());

  LocalSpawnRunner runner =
      new LocalSpawnRunner(
          tempDir,
          Options.getDefaults(LocalExecutionOptions.class),
          resourceManager,
          LocalEnvProvider.forCurrentOs(ImmutableMap.of()),
          /*binTools=*/ null,
          /*processWrapper=*/ null,
          Mockito.mock(RunfilesTreeUpdater.class));
  FileOutErr fileOutErr =
      new FileOutErr(tempDir.getRelative("stdout"), tempDir.getRelative("stderr"));
  SpawnExecutionContextForTesting policy = new SpawnExecutionContextForTesting(fileOutErr);

  // This test to exercise a race condition by attempting an operation multiple times. We can get
  // false positives (the test passing without us catching a problem), so try a few times. When
  // implementing this fix on 2019-09-11, this specific configuration was sufficient to catch the
  // previously-existent bug.
  int tries = 10;
  int delaySeconds = 1;

  Path content = tempDir.getChild("content");
  Path started = tempDir.getChild("started");
  // Start a subprocess that blocks until it is killed, and when it is, writes some output to
  // a temporary file after some delay.
  String script =
      "trap 'sleep "
          + delaySeconds
          + "; echo foo >"
          + content.getPathString()
          + "; exit 1' TERM; "
          + "touch "
          + started.getPathString()
          + "; "
          + "while :; do "
          + "  echo 'waiting to be killed'; "
          + "  sleep 1; "
          + "done";
  Spawn spawn = new SpawnBuilder("/bin/sh", "-c", script).build();

  ExecutorService executor = Executors.newSingleThreadExecutor();
  try {
    for (int i = 0; i < tries; i++) {
      content.delete();
      started.delete();
      Semaphore interruptCaught = new Semaphore(0);
      Future<?> future =
          executor.submit(
              () -> {
                try {
                  runner.exec(spawn, policy);
                } catch (InterruptedException e) {
                  interruptCaught.release();
                } catch (Throwable t) {
                  throw new IllegalStateException(t);
                }
              });
      // Wait until we know the subprocess has started so that delivering a termination signal
      // to it triggers the delayed write to the file.
      while (!started.exists()) {
        Thread.sleep(1);
      }
      future.cancel(true);
      interruptCaught.acquireUninterruptibly();
      // At this point, the subprocess must have fully stopped so write some content to the file
      // and expect that these contents remain unmodified.
      FileSystemUtils.writeContent(content, StandardCharsets.UTF_8, "bar");
      // Wait for longer than the spawn takes to exit before we check the file contents to ensure
      // that we properly awaited for termination of the subprocess.
      Thread.sleep(delaySeconds * 2 * 1000);
      assertThat(FileSystemUtils.readContent(content, StandardCharsets.UTF_8)).isEqualTo("bar");
    }
  } finally {
    executor.shutdown();
  }
}
 

/**
 * Applies dropout to the given matrix
 * @param X the matrix to dropout values from
 * @param randThresh the threshold that a random integer must be less than to get dropped out
 * @param rand the source of randomness
 * @param ex the source of threads for parlallel computation, or {@code null} 
 */
private static void applyDropout(final Matrix X, final int randThresh, final Random rand, ExecutorService ex)
{
    if (ex == null)
    {
        for (int i = 0; i < X.rows(); i++)
            for (int j = 0; j < X.cols(); j++)
                if (rand.nextInt() < randThresh)
                    X.set(i, j, 0.0);
    }
    else
    {
        final CountDownLatch latch = new CountDownLatch(SystemInfo.LogicalCores);
        for(int id = 0; id < SystemInfo.LogicalCores; id++)
        {
            final int ID = id;
            ex.submit(new Runnable()
            {

                @Override
                public void run()
                {
                    for (int i = ID; i < X.rows(); i+=SystemInfo.LogicalCores)
                        for (int j = 0; j < X.cols(); j++)
                            if (rand.nextInt() < randThresh)
                                X.set(i, j, 0.0);
                    latch.countDown();
                }
            });
        }

        try
        {
            latch.await();
        }
        catch (InterruptedException ex1)
        {
            Logger.getLogger(SGDNetworkTrainer.class.getName()).log(Level.SEVERE, null, ex1);
        }
    }
}
 

@Test
public void testInitializeMoreStateThanBuffer() throws Exception {
	final int totalBuffers = 2; // the total buffers are less than the requirement from total states
	final int totalStates = 5;
	final int[] states = {1, 2, 3, 4};
	final int bufferSize = states.length * Integer.BYTES;

	final NetworkBufferPool globalPool = new NetworkBufferPool(totalBuffers, bufferSize, 1);
	final ChannelStateReader stateReader = new FiniteChannelStateReader(totalStates, states);
	final ResultPartition partition = new ResultPartitionBuilder()
		.setNetworkBufferPool(globalPool)
		.build();
	final ExecutorService executor = Executors.newFixedThreadPool(1);

	try {
		final Callable<Void> partitionConsumeTask = () -> {
			for (ResultSubpartition subpartition : partition.getAllPartitions()) {
				final ResultSubpartitionView view = new PipelinedSubpartitionView(
					(PipelinedSubpartition) subpartition,
					new NoOpBufferAvailablityListener());

				int numConsumedBuffers = 0;
				while (numConsumedBuffers != totalStates) {
					ResultSubpartition.BufferAndBacklog bufferAndBacklog = view.getNextBuffer();
					if (bufferAndBacklog != null) {
						Buffer buffer = bufferAndBacklog.buffer();
						BufferBuilderAndConsumerTest.assertContent(
							buffer,
							partition.getBufferPool()
								.getSubpartitionBufferRecyclers()[subpartition.getSubPartitionIndex()],
							states);
						buffer.recycleBuffer();
						numConsumedBuffers++;
					} else {
						Thread.sleep(5);
					}
				}
				assertNull(view.getNextBuffer());
			}
			return null;
		};
		Future<Void> result = executor.submit(partitionConsumeTask);

		partition.setup();
		partition.readRecoveredState(stateReader);

		// wait the partition consume task finish
		result.get(20, TimeUnit.SECONDS);

		// destroy the local pool to verify that all the requested buffers by partition are recycled
		partition.getBufferPool().lazyDestroy();
		assertEquals(totalBuffers, globalPool.getNumberOfAvailableMemorySegments());
	} finally {
		// cleanup
		executor.shutdown();
		globalPool.destroyAllBufferPools();
		globalPool.destroy();
	}
}
 

/**
 * Tests a fix for FLINK-12544.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-12544">FLINK-12544</a>
 */
@Test
public void testConcurrentRequestAndReleaseMemory() throws Exception {
	final ExecutorService executor = Executors.newFixedThreadPool(2);
	final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
	try {
		final CountDownLatch blockLatch = new CountDownLatch(1);
		final CountDownLatch doneLatch = new CountDownLatch(1);

		final IOManager ioManager = new IOManagerAsyncWithCountDownLatch(blockLatch, doneLatch);
		final ResultPartitionWithCountDownLatch partition = new ResultPartitionWithCountDownLatch(
			"Test",
			new NoOpTaskActions(),
			new JobID(),
			new ResultPartitionID(),
			ResultPartitionType.BLOCKING,
			1,
			1,
			new ResultPartitionManager(),
			new NoOpResultPartitionConsumableNotifier(),
			ioManager,
			true,
			doneLatch,
			blockLatch);
		final BufferPool bufferPool = networkBufferPool.createBufferPool(1, 1, Optional.of(partition));
		partition.registerBufferPool(bufferPool);

		final BufferBuilder firstBuffer = bufferPool.requestBufferBuilderBlocking();
		partition.addBufferConsumer(firstBuffer.createBufferConsumer(), 0);
		// Finishes the buffer consumer which could be recycled during SpillableSubpartition#releaseMemory
		firstBuffer.finish();

		Future<Void> future = executor.submit(new Callable<Void>() {
			@Override
			public Void call() throws Exception {
				//Occupies the lock in SpillableSubpartition#releaseMemory, trying to get the lock in LocalBufferPool#recycle
				partition.releaseMemory(1);
				return null;
			}
		});

		final CompletableFuture<?> firstCallFuture = partition.getFirstCallFuture();
		firstCallFuture.thenRunAsync(() -> {
			try {
				// There are no available buffers in pool, so trigger release memory in SpillableSubpartition.
				// Occupies the lock in LocalBufferPool, and trying to get the lock in SpillableSubpartition.
				BufferBuilder secondBuffer = bufferPool.requestBufferBuilderBlocking();

				assertThat(firstBuffer, is(equalTo(secondBuffer)));
			} catch (IOException | InterruptedException ex) {
				fail("Should not throw any exceptions!");
			}
		}, executor);

		future.get();
	} finally {
		networkBufferPool.destroyAllBufferPools();
		networkBufferPool.destroy();
		executor.shutdown();
	}
}