java.util.concurrent.ExecutorCompletionService Java Examples

@Test
void example_parallel_completion_order() throws Exception {
    ExecutorService executor = Executors.newFixedThreadPool(10);

    CompletionService<Integer> completionService =
      new ExecutorCompletionService<>(executor);

    for (Integer integer : integers) {
        completionService.submit(() -> Utils.process(integer));
    }

    List<Integer> results = new ArrayList<>();

    for (int i = 0; i < integers.size(); i++) {
        results.add(completionService.take().get());
    }

    assertThat(results)
      .containsExactlyInAnyOrderElementsOf(integers);
}
 

/**
 * If poll returns non-null, the returned task is completed
 */
public void testPoll1() throws Exception {
    final ExecutorService e = Executors.newCachedThreadPool();
    final ExecutorCompletionService ecs = new ExecutorCompletionService(e);
    try (PoolCleaner cleaner = cleaner(e)) {
        assertNull(ecs.poll());
        Callable c = new StringTask();
        ecs.submit(c);

        long startTime = System.nanoTime();
        Future f;
        while ((f = ecs.poll()) == null) {
            if (millisElapsedSince(startTime) > LONG_DELAY_MS)
                fail("timed out");
            Thread.yield();
        }
        assertTrue(f.isDone());
        assertSame(TEST_STRING, f.get());
    }
}
 

@Test
public void testUpsertSelectSameBatchConcurrently() throws Exception {
	try (Connection conn = driver.connect(url, props)) {
	        int numUpsertSelectRunners = 5;
	        ExecutorService exec = Executors.newFixedThreadPool(numUpsertSelectRunners);
	        CompletionService<Boolean> completionService = new ExecutorCompletionService<Boolean>(exec);
	        List<Future<Boolean>> futures = Lists.newArrayListWithExpectedSize(numUpsertSelectRunners);
	        // run one UPSERT SELECT for 100 rows (that locks the rows for a long time)
	        futures.add(completionService.submit(new UpsertSelectRunner(dataTable, 0, 105, 1)));
	        // run four UPSERT SELECTS for 5 rows (that overlap with slow running UPSERT SELECT)
	        for (int i = 0; i < 100; i += 25) {
	            futures.add(completionService.submit(new UpsertSelectRunner(dataTable, i, i+25, 5)));
	        }
	        int received = 0;
	        while (received < futures.size()) {
	            Future<Boolean> resultFuture = completionService.take();
	            Boolean result = resultFuture.get();
	            received++;
	            assertTrue(result);
	        }
	        exec.shutdownNow();
	}
}
 

@Test
public void testMultipleClients() throws Exception {
  ExecutorService exec = Executors.newFixedThreadPool(NUM_THREADS);
  try {
    ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<>(exec);
    for (int i = 0; i < NUM_THREADS; ++i)
      ecs.submit(new IdLockTestThread("client_" + i));
    for (int i = 0; i < NUM_THREADS; ++i) {
      Future<Boolean> result = ecs.take();
      assertTrue(result.get());
    }
    idLock.assertMapEmpty();
  } finally {
    exec.shutdown();
    exec.awaitTermination(5000, TimeUnit.MILLISECONDS);
  }
}
 

@Test
@Timeout(TEST_TIMEOUT_SECONDS)
void testConcurrentlyCreateProxy() throws Exception {
	ExecutorService executorService = Executors.newFixedThreadPool(5);
	try {
		CompletionService<TestEntity> completionService = new ExecutorCompletionService<>(executorService);
		for (int x = 0; x < 50; x++) {
			TestEntity entity = new TestEntity(100 + x);
			int numRepetitions = 20;
			for (int i = 0; i < numRepetitions; i++) {
				completionService.submit(() -> ImmutableProxy.create(entity));
			}
			for (int i = 0; i < numRepetitions; i++) {
				TestEntity immutableProxy = completionService.take().get();
				assertThat(immutableProxy).isNotSameAs(entity);
				assertThat(immutableProxy.getNumber()).isEqualTo(entity.getNumber());
			}
			ImmutableProxy.clearCache();
		}
	} finally {
		executorService.shutdown();
		executorService.awaitTermination(TEST_TIMEOUT_SECONDS, TimeUnit.SECONDS);
	}
}
 

void solveAny(Executor e,
              Collection<Callable<Integer>> solvers)
    throws InterruptedException {
    CompletionService<Integer> cs
        = new ExecutorCompletionService<>(e);
    int n = solvers.size();
    List<Future<Integer>> futures = new ArrayList<>(n);
    Integer result = null;
    try {
        solvers.forEach(solver -> futures.add(cs.submit(solver)));
        for (int i = n; i > 0; i--) {
            try {
                Integer r = cs.take().get();
                if (r != null) {
                    result = r;
                    break;
                }
            } catch (ExecutionException ignore) {}
        }
    } finally {
        futures.forEach(future -> future.cancel(true));
    }

    if (result != null)
        use(result);
}
 

public ConcurrentTransferWorker(final SessionPool source,
                                final SessionPool destination,
                                final Transfer transfer,
                                final ThreadPool.Priority priority,
                                final TransferOptions options,
                                final TransferSpeedometer meter,
                                final TransferPrompt prompt,
                                final TransferErrorCallback error,
                                final ConnectionCallback connect,
                                final ProgressListener progressListener,
                                final StreamListener streamListener,
                                final NotificationService notification) {
    super(transfer, options, prompt, meter, error, progressListener, streamListener, connect, notification);
    this.source = source;
    this.destination = destination;
    this.pool = ThreadPoolFactory.get(String.format("%s-transfer", new AlphanumericRandomStringService().random()),
        transfer.getTransferType() == Host.TransferType.newconnection ? 1 : PreferencesFactory.get().getInteger("queue.connections.limit"), priority);
    this.completion = new ExecutorCompletionService<TransferStatus>(pool.executor());
}
 

/**
 * Runs {@code parallelism} instances of the {@code runners} in parallel. If one fails, no more
 * are scheduled, though inflight runs are not killed. The stdout for each command is written to a
 * file, and stderr is written to {@link System.err}.
 *
 * @param runners The commands to run
 * @param outputDirectory The directory to write stdout files to
 * @param parallelism The number of runs to do at a time
 * @return The paths to each of the output files
 * @throws StressorException If any of the processes fail or their output cannot be written to
 *     temporary files
 */
public List<Path> run(List<BuckRunner> runners, Path outputDirectory, int parallelism)
    throws StressorException {
  ExecutorService service = Executors.newFixedThreadPool(parallelism);
  ExecutorCompletionService<Integer> completionService = new ExecutorCompletionService<>(service);
  List<Path> filePaths =
      IntStream.range(0, runners.size())
          .mapToObj(i -> outputDirectory.resolve(Integer.toString(i) + ".log"))
          .collect(Collectors.toList());

  List<Future<Integer>> futures = queueRuns(completionService, filePaths, runners);
  String errorMessages = waitForFutures(completionService, futures);
  if (!errorMessages.isEmpty()) {
    throw new StressorException(errorMessages);
  }
  return filePaths;
}
 

private int executeRequests(String baseUrl, String[] timeSeriesStrings, int timesToRepeatTimeSeriesRequests,
    HttpClient httpClient, ExecutorCompletionService<MakeHttpRequestResult> completionService,
    List<Future<MakeHttpRequestResult>> futures) {
  Future<MakeHttpRequestResult> curFuture;
  String curUrl;
  int numRequests = 0;
  for (int j = 0; j < timesToRepeatTimeSeriesRequests; j++) {
    for (int i = 0; i < timeSeriesStrings.length; i++) {
      curUrl = baseUrl + timeSeriesStrings[i];
      curFuture =
          completionService.submit(new MakeHttpRequestCallable(httpClient, curUrl, i + j * timeSeriesStrings.length));
      numRequests++;
      futures.add(curFuture);
    }
  }
  return numRequests;
}
 

private List<Pair<String, Throwable>> execAndGetResults(List<QueryCallable> tasks)
        throws InterruptedException, java.util.concurrent.ExecutionException {
    ThreadPoolExecutor executor = new ThreadPoolExecutor(9//
            , 9 //
            , 1 //
            , TimeUnit.DAYS //
            , new LinkedBlockingQueue<Runnable>(100));
    CompletionService<Pair<String, Throwable>> service = new ExecutorCompletionService<>(executor);
    for (QueryCallable task : tasks) {
        service.submit(task);
    }

    List<Pair<String, Throwable>> results = new ArrayList<>();
    for (int i = 0; i < tasks.size(); i++) {
        Pair<String, Throwable> r = service.take().get();
        failFastIfNeeded(r);
        results.add(r);
    }
    executor.shutdown();
    return results;
}
 

private void waitForCompletion(ExecutorCompletionService<Boolean> completer, int numtasks) throws ExecutionException {
    int remainingTasks = numtasks;
    while (remainingTasks > 0) {
        try {
            Future<Boolean> completed = completer.poll(4, TimeUnit.SECONDS);
            if (completed == null) {
                System.err.println("Completer timed out");
                throw new RuntimeException(new TimeoutException());
            }
            else {
                completed.get();
            }
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        remainingTasks--;
    }
}
 

public GraphBasedSaga(EventStore eventStore,
    Executor executor,
    Map<String, SagaTask> tasks,
    SagaContext sagaContext,
    SingleLeafDirectedAcyclicGraph<SagaRequest> sagaTaskGraph) {

  this.eventStore = eventStore;
  this.tasks = tasks;

  this.transactionTaskRunner = new TaskRunner(
      traveller(sagaTaskGraph, new FromRootTraversalDirection<SagaRequest>()),
      new TransactionTaskConsumer(
          tasks,
          sagaContext,
          new ExecutorCompletionService<Operation>(executor)));

  this.sagaContext = sagaContext;
  this.compensationTaskRunner = new TaskRunner(
      traveller(sagaTaskGraph, new FromLeafTraversalDirection<SagaRequest>()),
      new CompensationTaskConsumer(tasks, sagaContext));

  currentTaskRunner = transactionTaskRunner;
}
 

/**
 * poll returns non-null when the returned task is completed
 */
public void testPoll1()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    assertNull(cs.poll());
    cs.submit(new StringTask());

    long startTime = System.nanoTime();
    Future f;
    while ((f = cs.poll()) == null) {
        if (millisElapsedSince(startTime) > LONG_DELAY_MS)
            fail("timed out");
        Thread.yield();
    }
    assertTrue(f.isDone());
    assertSame(TEST_STRING, f.get());
}
 

/**
 * timed poll returns non-null when the returned task is completed
 */
public void testPoll2()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    assertNull(cs.poll());
    cs.submit(new StringTask());

    long startTime = System.nanoTime();
    Future f;
    while ((f = cs.poll(SHORT_DELAY_MS, MILLISECONDS)) == null) {
        if (millisElapsedSince(startTime) > LONG_DELAY_MS)
            fail("timed out");
        Thread.yield();
    }
    assertTrue(f.isDone());
    assertSame(TEST_STRING, f.get());
}
 

@Test
public void testCompletionServiceRunnableCaptures() throws InterruptedException, Exception {
    // Setup
    ExecutorService executor = Executors.newCachedThreadPool();
    CompletionService<Object> delegate = new ExecutorCompletionService<>(executor);
    CompletionService<Object> cs = StateCapture.capturingDecorator(delegate);

    CapturedState mockCapturedState = mock(CapturedState.class);
    Runnable mockRunnable = mock(Runnable.class);
    ThreadLocalStateCaptor.THREAD_LOCAL.set(mockCapturedState);
    Object result = new Object();
    Future<Object> futureResult = cs.submit(mockRunnable, result);
    assertThat("Expected the delegate response to return",
        result, sameInstance(futureResult.get()));
    executor.shutdown();

    verifyStandardCaptures(mockCapturedState, mockRunnable);
}
 

/**
 * Executes one or more tasks that is needed by this state.
 *
 * @param executor -  ExecutorService
 */
@Override
public void execute(ExecutorService executor) {
  ecs = new ExecutorCompletionService<>(executor);
  for (EndpointStateMachine endpoint : connectionManager.getValues()) {
    Callable<EndPointStates> endpointTask = getEndPointTask(endpoint);
    if (endpointTask != null) {
      ecs.submit(endpointTask);
    } else {
      // This can happen if a task is taking more time than the timeOut
      // specified for the task in await, and when it is completed the task
      // has set the state to Shutdown, we may see the state as shutdown
      // here. So, we need to Shutdown DatanodeStateMachine.
      LOG.error("State is Shutdown in RunningDatanodeState");
      context.setState(DatanodeStateMachine.DatanodeStates.SHUTDOWN);
    }
  }
}
 

public BlockingQueueConsumer(int numThreads) {
    this.queue = new ArrayBlockingQueue<DynamoDBEntryWithSize>(20);
    int numProcessors = Runtime.getRuntime().availableProcessors();
    if (numProcessors > numThreads) {
        numThreads = numProcessors;
    }
    this.threadPool = Executors.newFixedThreadPool(numThreads);
    this.exec = new ExecutorCompletionService<Void>(threadPool);
}
 

void configReloaded(ApplicationId applicationId) {
    List<DelayedConfigResponses.DelayedConfigResponse> responses = delayedConfigResponses.drainQueue(applicationId);
    String logPre = TenantRepository.logPre(applicationId);
    if (log.isLoggable(Level.FINE)) {
        log.log(Level.FINE, logPre + "Start of configReload: " + responses.size() + " requests on delayed requests queue");
    }
    int responsesSent = 0;
    CompletionService<Boolean> completionService = new ExecutorCompletionService<>(executorService);
    while (!responses.isEmpty()) {
        DelayedConfigResponses.DelayedConfigResponse delayedConfigResponse = responses.remove(0);
        // Discard the ones that we have already answered
        // Doing cancel here deals with the case where the timer is already running or has not run, so
        // there is no need for any extra check.
        if (delayedConfigResponse.cancel()) {
            if (log.isLoggable(Level.FINE)) {
                logRequestDebug(Level.FINE, logPre + "Timer cancelled for ", delayedConfigResponse.request);
            }
            // Do not wait for this request if we were unable to execute
            if (addToRequestQueue(delayedConfigResponse.request, false, completionService)) {
                responsesSent++;
            }
        } else {
            log.log(Level.FINE, logPre + "Timer already cancelled or finished or never scheduled");
        }
    }

    for (int i = 0; i < responsesSent; i++) {
        try {
            completionService.take();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
        }
    }

    log.log(Level.FINE, logPre + "Finished reloading " + responsesSent + " requests");
}
 

/**
 * Public constructor.
 * @param cloudDestination the cloud destination to use.
 * @param partitions the set of partitions to compact.
 * @param vcrMetrics the metrics to update.
 */
public CloudStorageCompactor(CloudDestination cloudDestination, CloudConfig cloudConfig, Set<PartitionId> partitions,
    VcrMetrics vcrMetrics) {
  this.cloudDestination = cloudDestination;
  this.partitions = partitions;
  this.vcrMetrics = vcrMetrics;
  this.shutDownTimeoutSecs = cloudConfig.cloudBlobCompactionShutdownTimeoutSecs;
  this.numThreads = cloudConfig.cloudCompactionNumThreads;
  compactionTimeLimitMs = TimeUnit.HOURS.toMillis(cloudConfig.cloudBlobCompactionIntervalHours);
  logger.info("Allocating {} threads for compaction", numThreads);
  executorService = Executors.newFixedThreadPool(numThreads);
  executorCompletionService = new ExecutorCompletionService<>(executorService);
  doneLatch.set(new CountDownLatch(0));
}
 

/**
 * Take returns the same future object returned by submit
 */
public void testTake2() throws InterruptedException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    Future f1 = cs.submit(new StringTask());
    Future f2 = cs.take();
    assertSame(f1, f2);
}
 

@Override
public Pair<byte[][], int[]> getChunk(List<Integer> redisIds, String cacheKey, long chunkPos, int chunkSize) throws
                                                       ECFileCacheException {

  List<DecoratedJedisPool> jedisPools = getJedisPools(redisIds);

  byte[][] redisDataArray = new byte[jedisPools.size()][];

  CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(pool);
  List<Future<Integer>> futures = new ArrayList<Future<Integer>>();

  String field = chunkPos + SEP + chunkSize;

  int failCount = 0;
  for (int i = 0; i < jedisPools.size(); ++i) {
    DecoratedJedisPool jedis = jedisPools.get(i);
    if (jedis != null) {
      String key = cacheKey + SEP + i;
      RedisGetChunk redisGetChunk = new RedisGetChunk(jedis, key, field, redisDataArray, i);

      if (!pool.isShutdown()) {
        Future<Integer> future = completionService.submit(redisGetChunk);
        futures.add(future);
      }
    } else {
      failCount++;
    }
  }
  checkRedisResult(completionService, futures, failCount);

  return convertChunk(redisDataArray, chunkSize);
}
 

private long executeQueries(final List<String> queries, final Context ctx) throws Exception {
    int maxThreads = KylinConfig.getInstanceFromEnv().getMigrationRuleQueryLatencyMaxThreads();
    int threadNum = Math.min(maxThreads, queries.size());
    ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
    CompletionService<Long> completionService = new ExecutorCompletionService<Long>(threadPool);
    final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
    long start = System.currentTimeMillis();
    for (final String query : queries) {
        completionService.submit(new Callable<Long>() {
            @Override
            public Long call() throws Exception {
                SecurityContextHolder.getContext().setAuthentication(auth);
                SQLRequest sqlRequest = new SQLRequest();
                sqlRequest.setProject(ctx.getSrcProjectName());
                sqlRequest.setSql(query);
                SQLResponse sqlResponse = ctx.getQueryService().doQueryWithCache(sqlRequest, false);
                if (sqlResponse.getIsException()) {
                    throw new RuleValidationException(sqlResponse.getExceptionMessage());
                }
                return sqlResponse.getDuration();
            }

        });
    }
    long timeCostSum = 0L;
    for (int i = 0; i < queries.size(); ++i) {
        try {
            timeCostSum += completionService.take().get();
        } catch (InterruptedException | ExecutionException e) {
            threadPool.shutdownNow();
            throw e;
        }
    }
    long end = System.currentTimeMillis();
    logger.info("Execute" + queries.size() + " queries took " + (end - start) + " ms, query time cost sum "
            + timeCostSum + " ms.");
    return timeCostSum / queries.size();
}
 

public IndexScanIterator(Snapshot snapshot, TiDAGRequest req, Iterator<Long> handleIterator) {
  TiSession session = snapshot.getSession();
  TiConfiguration conf = session.getConf();
  this.dagReq = req;
  this.handleIterator = handleIterator;
  this.snapshot = snapshot;
  this.batchSize = conf.getIndexScanBatchSize();
  this.completionService = new ExecutorCompletionService<>(session.getThreadPoolForIndexScan());
}
 

/**
 * ecs.submit(null) throws NullPointerException
 */
public void testSubmitNullCallable() {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    try {
        cs.submit((Callable) null);
        shouldThrow();
    } catch (NullPointerException success) {}
}
 

/**
 * Main API to run both production and consumption.
 */
public E execute() {
  try {
    ExecutorCompletionService<Boolean> producerService = startProducers();
    Future<E> future = startConsumer();
    // Wait for consumer to be done
    return future.get();
  } catch (Exception e) {
    throw new HoodieException(e);
  }
}
 

FlushTableSubprocedurePool(String name, Configuration conf, Abortable abortable) {
  this.abortable = abortable;
  // configure the executor service
  long keepAlive = conf.getLong(
    RegionServerFlushTableProcedureManager.FLUSH_TIMEOUT_MILLIS_KEY,
    RegionServerFlushTableProcedureManager.FLUSH_TIMEOUT_MILLIS_DEFAULT);
  int threads = conf.getInt(CONCURENT_FLUSH_TASKS_KEY, DEFAULT_CONCURRENT_FLUSH_TASKS);
  this.name = name;
  executor = Threads.getBoundedCachedThreadPool(threads, keepAlive, TimeUnit.MILLISECONDS,
      "rs(" + name + ")-flush-proc");
  taskPool = new ExecutorCompletionService<>(executor);
}
 

@Override
@NonNull
public Pair<Boolean,Collection<? extends URL>> execute(
        @NonNull final Function<URL,Boolean> fnc,
        @NonNull final Callable<Boolean> cancel,
        @NonNull final Collection<? extends URL> binaries) {
    final CompletionService<URL> cs = new ExecutorCompletionService<URL>(RP);
    int submitted = 0;
    for (URL binary : binaries) {
        cs.submit(new Task(binary,fnc, cancel));
        submitted++;
    }
    final Collection<URL> result = new ArrayDeque<URL>();
    //Don't break the cycle when is canceled,
    //rather wait for all submitted task, they should die fast.
    //The break will cause logging of wrong number of scanned roots.
    for (int i=0; i< submitted; i++) {
        try {                    
            final Future<URL> becomeURL = cs.take();
            final URL url = becomeURL.get();
            if (url != null) {
                result.add(url);
            }
        } catch (Exception ex) {
            Exceptions.printStackTrace(ex);
        }
    }
    boolean success;
    try {
        success = !cancel.call();
    } catch (Exception e) {
        Exceptions.printStackTrace(e);
        success = false;
    }
    LOG.log(Level.FINER, "Canceled: {0}", !success);  //NOI18N
    return Pair.<Boolean,Collection<? extends URL>>of(success,result);
}
 

@Override
public ReconfigurationPlan solve(Parameters cra, Instance orig) throws SchedulerException {
    stats = new StaticPartitioningStatistics(cra, orig, System.currentTimeMillis(), workersCount);
    long d = -System.currentTimeMillis();
    List<Instance> partitions = split(cra, orig);
    d += System.currentTimeMillis();

    stats.setSplittingStatistics(partitions.size(), d);
    ExecutorService exe = Executors.newFixedThreadPool(this.workersCount);
    CompletionService<SolvingStatistics> completionService = new ExecutorCompletionService<>(exe);
    List<SolvingStatistics> results = new ArrayList<>(partitions.size());


    runners = new ArrayList<>();
    long duration = -System.currentTimeMillis();
    for (Instance partition : partitions) {
        InstanceSolverRunner runner = new InstanceSolverRunner(cra, partition);
        completionService.submit(runner);
        runners.add(runner);
    }

    for (int i = 0; i < partitions.size(); i++) {
        try {
            results.add(completionService.take().get());
        } catch (ExecutionException ignore) {
            Throwable cause = ignore.getCause();
            if (cause != null) {
                throw new SplitException(null, cause.getMessage(), ignore);
            }
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new SplitException(orig.getModel(), e.getMessage(), e);
        }
    }
    duration += System.currentTimeMillis();
    stats.setSolvingDuration(duration);
    exe.shutdown();

    return merge(orig, results);
}
 

public ConcurrentScheduler() {

		int threadCount = AutomationEngine.getInstance().getTestAgenda().getThreadCount();
		if (threadCount <= 0) {
			NTHREADS = 1;
		} else {
			NTHREADS = threadCount;
		}
		execService = Executors.newFixedThreadPool(NTHREADS);
		completionService = new ExecutorCompletionService<String>(execService);
		digraph = AutomationEngine.getInstance().getTestDigraph();
		
	}
 

@Override
public Boolean execute(I context, Collection<IProcessor<I, Boolean>> processors) throws InterruptedException, ExecutionException {
	Boolean results = true;

	if (!processors.isEmpty()) {
		ThreadPoolExecutor threadPoolExecutor = ThreadPoolFactory.newFixedThreadPool(parallelCount);
		CompletionService<Boolean> cService = new ExecutorCompletionService<>(threadPoolExecutor);
		results = executeByServiceThreadPool(cService, context, processors);
		threadPoolExecutor.shutdown();
		threadPoolExecutor.awaitTermination(11, TimeUnit.HOURS);
	}
	return results;
}