java.util.concurrent.CompletionService Java Examples

public void testPoolBatch() throws Exception {
    PbrpcClient client = PbrpcClientFactory.buildPooledConnection(new PooledConfiguration(),
            "127.0.0.1", 8088, 60000);
    int multiSize = 8;
    int totalRequestSize = 100;
    ExecutorService pool = Executors.newFixedThreadPool(multiSize);
    CompletionService<DemoBatchResponse> completionService = new ExecutorCompletionService<DemoBatchResponse>(
            pool);

    BatchInvoker invoker = new BatchInvoker(client);
    long time = System.currentTimeMillis();
    for (int i = 0; i < totalRequestSize; i++) {
        completionService.submit(invoker);
    }

    for (int i = 0; i < totalRequestSize; i++) {
        completionService.take().get();
    }

    long timetook = System.currentTimeMillis() - time;
    LOG.info("Total using " + timetook + "ms");
    LOG.info("QPS:" + 1000f / ((timetook) / (1.0f * totalRequestSize)));
}
 

@Test
@Timeout(30)
void testConcurrentlyCreateProxyClasses() throws Exception {
	ExecutorService executorService = Executors.newFixedThreadPool(4);
	try {
		CompletionService<Void> completionService = new ExecutorCompletionService<>(executorService);
		for (int i = 0; i < 4; i++) {
			completionService.submit(() -> {
				for (int r = 0; r < 100; r++) {
					PropertyUtils.getPropertyDescriptor(TestEntity.class, TestEntity::getNumber);
					PropertyUtils.clearCache();
				}
				return null;
			});
		}
		for (int i = 0; i < 4; i++) {
			completionService.take().get();
		}
	} finally {
		executorService.shutdown();
	}
}
 

@Test
void example_parallel_completion_order() throws Exception {
    ExecutorService executor = Executors.newFixedThreadPool(10);

    CompletionService<Integer> completionService =
      new ExecutorCompletionService<>(executor);

    for (Integer integer : integers) {
        completionService.submit(() -> Utils.process(integer));
    }

    List<Integer> results = new ArrayList<>();

    for (int i = 0; i < integers.size(); i++) {
        results.add(completionService.take().get());
    }

    assertThat(results)
      .containsExactlyInAnyOrderElementsOf(integers);
}
 

@Test
public void testDoSmth() throws Exception {
    Demo.DemoRequest.Builder req = Demo.DemoRequest.newBuilder();
    req.setUserId(1);

    int multiSize = 12;
    int totalRequestSize = 10;
    ExecutorService pool = Executors.newFixedThreadPool(multiSize);
    CompletionService<Demo.DemoResponse> completionService = new ExecutorCompletionService<Demo.DemoResponse>(
            pool);

    Invoker invoker = new Invoker(req.build());
    long time = System.currentTimeMillis();
    for (int i = 0; i < totalRequestSize; i++) {
        completionService.submit(invoker);
    }

    for (int i = 0; i < totalRequestSize; i++) {
        completionService.take().get();
    }

    long timetook = System.currentTimeMillis() - time;
    System.out.println("Total using " + timetook + "ms");
    System.out.println("QPS:" + 1000f / ((timetook) / (1.0f * totalRequestSize)));
}
 

/**
 * Test that LoggingRegistry is concurrent-sage initialized over multiple calls. Creating more than 1000 threads can
 * cause significant performance impact.
 *
 * @throws InterruptedException
 * @throws ExecutionException
 */
@Test( timeout = 30000 )
public void testLoggingRegistryConcurrentInitialization() throws InterruptedException, ExecutionException {
  CountDownLatch start = new CountDownLatch( 1 );

  int count = 10;
  CompletionService<LoggingRegistry> drover = registerHounds( count, start );
  // fire!
  start.countDown();

  Set<LoggingRegistry> distinct = new HashSet<LoggingRegistry>();

  int i = 0;
  while ( i < count ) {
    Future<LoggingRegistry> complete = drover.poll( 15, TimeUnit.SECONDS );
    LoggingRegistry instance = complete.get();
    distinct.add( instance );
    i++;
  }
  Assert.assertEquals( "Only one singlton instance ;)", 1, distinct.size() );
}
 

@Test
public void testCompletionServiceRunnableCaptures() throws InterruptedException, Exception {
    // Setup
    ExecutorService executor = Executors.newCachedThreadPool();
    CompletionService<Object> delegate = new ExecutorCompletionService<>(executor);
    CompletionService<Object> cs = StateCapture.capturingDecorator(delegate);

    CapturedState mockCapturedState = mock(CapturedState.class);
    Runnable mockRunnable = mock(Runnable.class);
    ThreadLocalStateCaptor.THREAD_LOCAL.set(mockCapturedState);
    Object result = new Object();
    Future<Object> futureResult = cs.submit(mockRunnable, result);
    assertThat("Expected the delegate response to return",
        result, sameInstance(futureResult.get()));
    executor.shutdown();

    verifyStandardCaptures(mockCapturedState, mockRunnable);
}
 

/**
 * @param states This is a collection of TaskState.
 */
@Override
public void publishData(Collection<? extends WorkUnitState> states)
    throws IOException {
  CompletionService<Collection<HiveSpec>> completionService =
      new ExecutorCompletionService<>(this.hivePolicyExecutor);

  int toRegisterPathCount = computeSpecs(states, completionService);
  for (int i = 0; i < toRegisterPathCount; i++) {
    try {
      for (HiveSpec spec : completionService.take().get()) {
        allRegisteredPartitions.add(spec);
        this.hiveRegister.register(spec);
      }
    } catch (InterruptedException | ExecutionException e) {
      log.info("Failed to generate HiveSpec", e);
      throw new IOException(e);
    }
  }
  log.info("Finished registering all HiveSpecs");
}
 

/**
 * GET urls in parallel using the executor service.
 * @param urls absolute URLs to GET
 * @param timeoutMs timeout in milliseconds for each GET request
 * @return instance of CompletionService. Completion service will provide
 *   results as they arrive. The order is NOT same as the order of URLs
 */
public CompletionService<GetMethod> execute(List<String> urls, int timeoutMs) {
  HttpClientParams clientParams = new HttpClientParams();
  clientParams.setConnectionManagerTimeout(timeoutMs);
  HttpClient client = new HttpClient(clientParams, _connectionManager);

  CompletionService<GetMethod> completionService = new ExecutorCompletionService<>(_executor);
  for (String url : urls) {
    completionService.submit(() -> {
      try {
        GetMethod getMethod = new GetMethod(url);
        getMethod.getParams().setSoTimeout(timeoutMs);
        client.executeMethod(getMethod);
        return getMethod;
      } catch (Exception e) {
        // Log only exception type and message instead of the whole stack trace
        LOGGER.warn("Caught '{}' while executing GET on URL: {}", e.toString(), url);
        throw e;
      }
    });
  }
  return completionService;
}
 

/**
 * poll returns non-null when the returned task is completed
 */
public void testPoll1()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    assertNull(cs.poll());
    cs.submit(new StringTask());

    long startTime = System.nanoTime();
    Future f;
    while ((f = cs.poll()) == null) {
        if (millisElapsedSince(startTime) > LONG_DELAY_MS)
            fail("timed out");
        Thread.yield();
    }
    assertTrue(f.isDone());
    assertSame(TEST_STRING, f.get());
}
 

/**
 * timed poll returns non-null when the returned task is completed
 */
public void testPoll2()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    assertNull(cs.poll());
    cs.submit(new StringTask());

    long startTime = System.nanoTime();
    Future f;
    while ((f = cs.poll(SHORT_DELAY_MS, MILLISECONDS)) == null) {
        if (millisElapsedSince(startTime) > LONG_DELAY_MS)
            fail("timed out");
        Thread.yield();
    }
    assertTrue(f.isDone());
    assertSame(TEST_STRING, f.get());
}
 

void solveAny(Executor e,
              Collection<Callable<Integer>> solvers)
    throws InterruptedException {
    CompletionService<Integer> cs
        = new ExecutorCompletionService<>(e);
    int n = solvers.size();
    List<Future<Integer>> futures = new ArrayList<>(n);
    Integer result = null;
    try {
        solvers.forEach(solver -> futures.add(cs.submit(solver)));
        for (int i = n; i > 0; i--) {
            try {
                Integer r = cs.take().get();
                if (r != null) {
                    result = r;
                    break;
                }
            } catch (ExecutionException ignore) {}
        }
    } finally {
        futures.forEach(future -> future.cancel(true));
    }

    if (result != null)
        use(result);
}
 

@Test
public void testMultiThreadRawLookupOrStartFindsActorPreviouslyStartedWIthRawLookupOrStart() {
  final int size = 1000;

  List<Address> addresses = IntStream.range(0, size)
      .mapToObj((ignored) -> world.addressFactory().unique())
      .collect(Collectors.toList());

  CompletionService<Actor> completionService =
      new ExecutorCompletionService<>(exec);

  final Definition definition = Definition.has(ParentInterfaceActor.class,
      ParentInterfaceActor::new);

  multithreadedLookupOrStartTest(index ->
          completionService.submit(() ->
              world.stage()
                  .rawLookupOrStart(definition, addresses.get(index)))
      , size);
}
 

private ByteBuffer getFirstToComplete(
    CompletionService<ByteBuffer> hedgedService,
    ArrayList<Future<ByteBuffer>> futures) throws InterruptedException {
  if (futures.isEmpty()) {
    throw new InterruptedException("let's retry");
  }
  Future<ByteBuffer> future = null;
  try {
    future = hedgedService.take();
    ByteBuffer bb = future.get();
    futures.remove(future);
    return bb;
  } catch (ExecutionException e) {
    // already logged in the Callable
    futures.remove(future);
  } catch (CancellationException ce) {
    // already logged in the Callable
    futures.remove(future);
  }

  throw new InterruptedException("let's retry");
}
 

@Test
@SuppressWarnings("unchecked") // mocked generics
public void testContainerLocalizerClosesFilesystems() throws Exception {
  // verify filesystems are closed when localizer doesn't fail
  FileContext fs = FileContext.getLocalFSFileContext();
  spylfs = spy(fs.getDefaultFileSystem());
  ContainerLocalizer localizer = setupContainerLocalizerForTest();
  doNothing().when(localizer).localizeFiles(any(LocalizationProtocol.class),
      any(CompletionService.class), any(UserGroupInformation.class));
  verify(localizer, never()).closeFileSystems(
      any(UserGroupInformation.class));
  localizer.runLocalization(nmAddr);
  verify(localizer).closeFileSystems(any(UserGroupInformation.class));

  spylfs = spy(fs.getDefaultFileSystem());
  // verify filesystems are closed when localizer fails
  localizer = setupContainerLocalizerForTest();
  doThrow(new YarnRuntimeException("Forced Failure")).when(localizer).localizeFiles(
      any(LocalizationProtocol.class), any(CompletionService.class),
      any(UserGroupInformation.class));
  verify(localizer, never()).closeFileSystems(
      any(UserGroupInformation.class));
  localizer.runLocalization(nmAddr);
  verify(localizer).closeFileSystems(any(UserGroupInformation.class));
}
 

private ByteBuffer getFirstToComplete(
    CompletionService<ByteBuffer> hedgedService,
    ArrayList<Future<ByteBuffer>> futures) throws InterruptedException {
  if (futures.isEmpty()) {
    throw new InterruptedException("let's retry");
  }
  Future<ByteBuffer> future = null;
  try {
    future = hedgedService.take();
    ByteBuffer bb = future.get();
    futures.remove(future);
    return bb;
  } catch (ExecutionException e) {
    // already logged in the Callable
    futures.remove(future);
  } catch (CancellationException ce) {
    // already logged in the Callable
    futures.remove(future);
  }

  throw new InterruptedException("let's retry");
}
 

/**
 * Test that LoggingRegistry is concurrent-sage initialized over multiple calls. Creating more than 1000 threads can
 * cause significant performance impact.
 * 
 * @throws InterruptedException
 * @throws ExecutionException
 * 
 */
@Test( timeout = 30000 )
public void testLoggingRegistryConcurrentInitialization() throws InterruptedException, ExecutionException {
  CountDownLatch start = new CountDownLatch( 1 );

  int count = 10;
  CompletionService<LoggingRegistry> drover = registerHounds( count, start );
  // fire!
  start.countDown();

  Set<LoggingRegistry> distinct = new HashSet<LoggingRegistry>();

  int i = 0;
  while ( i < count ) {
    Future<LoggingRegistry> complete = drover.poll( 15, TimeUnit.SECONDS );
    LoggingRegistry instance = complete.get();
    distinct.add( instance );
    i++;
  }
  Assert.assertEquals( "Only one singlton instance ;)", 1, distinct.size() );
}
 

/**
 * successful and failed tasks are both returned
 */
public void testTaskAssortment()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    ArithmeticException ex = new ArithmeticException();
    for (int i = 0; i < 2; i++) {
        cs.submit(new StringTask());
        cs.submit(callableThrowing(ex));
        cs.submit(runnableThrowing(ex), null);
    }
    int normalCompletions = 0;
    int exceptionalCompletions = 0;
    for (int i = 0; i < 3 * 2; i++) {
        try {
            if (cs.take().get() == TEST_STRING)
                normalCompletions++;
        }
        catch (ExecutionException expected) {
            assertTrue(expected.getCause() instanceof ArithmeticException);
            exceptionalCompletions++;
        }
    }
    assertEquals(2 * 1, normalCompletions);
    assertEquals(2 * 2, exceptionalCompletions);
    assertNull(cs.poll());
}
 

/**
 * poll returns null before the returned task is completed
 */
public void testPollReturnsNull()
    throws InterruptedException, ExecutionException {
    CompletionService cs = new ExecutorCompletionService(cachedThreadPool);
    final CountDownLatch proceed = new CountDownLatch(1);
    cs.submit(new Callable() { public String call() throws Exception {
        proceed.await();
        return TEST_STRING;
    }});
    assertNull(cs.poll());
    assertNull(cs.poll(0L, MILLISECONDS));
    assertNull(cs.poll(Long.MIN_VALUE, MILLISECONDS));
    long startTime = System.nanoTime();
    assertNull(cs.poll(timeoutMillis(), MILLISECONDS));
    assertTrue(millisElapsedSince(startTime) >= timeoutMillis());
    proceed.countDown();
    assertSame(TEST_STRING, cs.take().get());
}
 

protected Boolean executeByServiceThreadPool(CompletionService<Boolean> cService, I context, Collection<IProcessor<I, Boolean>> processors) throws InterruptedException, ExecutionException {
	Boolean results = true;
	for (IProcessor<I, Boolean> processor : processors) {
		cService.submit(new Callable<Boolean>() {
			@Override
			public Boolean call() throws Exception {
				if (processor.isEnabled()) {
					try {
						return processor.process(context);
					} catch (Exception ex) {
						logger.error("executeByServiceThreadPool", ex);
						return false;
					}
				} else {
					logger.info(String.format("processor:%s,%s", processor.getClass().getName(), processor.isEnabled()));
				}
				return true;
			}
		});
	}
	for (int i = 0; i < processors.size(); i++) {
		results = results && cService.take().get();
		if (!results) {
			break;
		}
	}
	return results;
}
 

@Test
@SuppressWarnings("unchecked")
public void testLocalizerTokenIsGettingRemoved() throws Exception {
  FileContext fs = FileContext.getLocalFSFileContext();
  spylfs = spy(fs.getDefaultFileSystem());
  ContainerLocalizer localizer = setupContainerLocalizerForTest();
  doNothing().when(localizer).localizeFiles(any(LocalizationProtocol.class),
      any(CompletionService.class), any(UserGroupInformation.class));
  localizer.runLocalization(nmAddr);
  verify(spylfs, times(1)).delete(tokenPath, false);
}
 

@Override
public Map<Long, Integer> getChunkPosAndSize(List<Integer> redisIds, String cacheKey) throws ECFileCacheException {

  List<DecoratedJedisPool> jedisPools = getJedisPools(redisIds);
  @SuppressWarnings("unchecked")
  Set<byte[]>[] redisFields = new Set[jedisPools.size()];

  CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(pool);
  List<Future<Integer>> futures = new ArrayList<Future<Integer>>();

  int failCount = 0;
  for (int i = 0; i < jedisPools.size(); ++i) {

    DecoratedJedisPool jedis = jedisPools.get(i);
    if (jedis != null) {
      final String key = cacheKey + SEP + Integer.toString(i);
      RedisHKeys redisHKeys = new RedisHKeys(jedis, key, redisFields, i);

      if (!pool.isShutdown()) {
        Future<Integer> future = completionService.submit(redisHKeys);
        futures.add(future);
      }
    } else {
      failCount++;
    }
  }
  checkRedisResult(completionService, futures, failCount);

  return convertChunkPosAndSize(redisFields);
}
 

@Override
public Pair<byte[][], int[]> getChunk(List<Integer> redisIds, String cacheKey, long chunkPos, int chunkSize) throws
                                                       ECFileCacheException {

  List<DecoratedJedisPool> jedisPools = getJedisPools(redisIds);

  byte[][] redisDataArray = new byte[jedisPools.size()][];

  CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(pool);
  List<Future<Integer>> futures = new ArrayList<Future<Integer>>();

  String field = chunkPos + SEP + chunkSize;

  int failCount = 0;
  for (int i = 0; i < jedisPools.size(); ++i) {
    DecoratedJedisPool jedis = jedisPools.get(i);
    if (jedis != null) {
      String key = cacheKey + SEP + i;
      RedisGetChunk redisGetChunk = new RedisGetChunk(jedis, key, field, redisDataArray, i);

      if (!pool.isShutdown()) {
        Future<Integer> future = completionService.submit(redisGetChunk);
        futures.add(future);
      }
    } else {
      failCount++;
    }
  }
  checkRedisResult(completionService, futures, failCount);

  return convertChunk(redisDataArray, chunkSize);
}
 

@Override
public List<Pair<byte[][], int[]>> get(List<Integer> redisIds, String cacheKey) throws ECFileCacheException {

  List<DecoratedJedisPool> jedisPools = getJedisPools(redisIds);

  @SuppressWarnings("unchecked")
  Map<byte[], byte[]>[] redisDataList = new Map[redisIds.size()];

  CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(pool);
  List<Future<Integer>> futures = new ArrayList<Future<Integer>>();

  int failCount = 0;
  for (int i = 0; i < jedisPools.size(); ++i) {
    DecoratedJedisPool jedis = jedisPools.get(i);
    if (jedis != null) {
      final String key = cacheKey + SEP + i;
      RedisGetAll redisGetAll = new RedisGetAll(jedis, key, redisDataList, i);

      if (!pool.isShutdown()) {
        Future<Integer> future = completionService.submit(redisGetAll);
        futures.add(future);
      }
    } else {
      failCount++;
    }
  }
  checkRedisResult(completionService, futures, failCount);

  return convert(redisDataList);
}
 

@Override
public void put(List<Integer> redisIds, String cacheKey, long chunkPos, final byte[][] chunks) throws ECFileCacheException {

  long dataLength = checkDataAndGetLength(chunks);
  List<DecoratedJedisPool> jedisPools = getJedisPools(redisIds);

  CompletionService<Integer> completionService = new ExecutorCompletionService<Integer>(pool);
  List<Future<Integer>> futures = new ArrayList<Future<Integer>>();

  int failCount = 0;
  for (int i = 0; i < jedisPools.size(); ++i) {
    DecoratedJedisPool jedis = jedisPools.get(i);
    if (jedis != null) {
      final String key = cacheKey + SEP + i;
      final String field = chunkPos + SEP + dataLength;
      final byte[] data = chunks[i];
      RedisPutChunk redisPutChunk = new RedisPutChunk(jedis, key, field, data);

      if (!pool.isShutdown()) {
        Future<Integer> future = completionService.submit(redisPutChunk);
        futures.add(future);
      }
    } else {
      failCount++;
    }
  }
  checkRedisResult(completionService, futures, failCount);
}
 

void solveAll(Executor e,
              Collection<Callable<Integer>> solvers)
    throws InterruptedException, ExecutionException {
    CompletionService<Integer> cs
        = new ExecutorCompletionService<>(e);
    solvers.forEach(cs::submit);
    for (int i = solvers.size(); i > 0; i--) {
        Integer r = cs.take().get();
        if (r != null)
            use(r);
    }
}
 

@Test(timeout = 11000)
public void fail_fast() throws InterruptedException, ExecutionException {
  ExecutorService executorService = Executors.newFixedThreadPool(2);
  CompletionService<Long> completionService = new ExecutorCompletionService<Long>(executorService);

  BlockingQueue<OWLCompositeObject> queue = new LinkedBlockingQueue<OWLCompositeObject>();
  BlockingQueue<OntologySetup> ontologyQueue = new LinkedBlockingQueue<OntologySetup>();
  OwlOntologyProducer producer =
      new OwlOntologyProducer(queue, ontologyQueue, new AtomicInteger(), graph);
  OntologySetup ontologyConfig = new OntologySetup();

  ontologyConfig.setUrl("http://localhost:10000/foo.owl");

  List<Future<?>> futures = new ArrayList<>();
  futures.add(completionService.submit(producer));
  futures.add(completionService.submit(producer));
  Thread.sleep(1000);
  ontologyQueue.put(ontologyConfig);

  expectedException.expect(ExecutionException.class);
  while (futures.size() > 0) {
    Future<?> completedFuture = completionService.take();
    futures.remove(completedFuture);
    completedFuture.get();
  }

  executorService.shutdown();
  executorService.awaitTermination(10, TimeUnit.SECONDS);

}
 

private static <T> T takeUnchecked(CompletionService<T> completionService)
        throws InterruptedException
{
    try {
        return completionService.take().get();
    }
    catch (ExecutionException e) {
        throw new RuntimeException(e);
    }
}
 

public void doWork() throws Exception{
    CompletionService<String> completionService = new ExecutorCompletionService<String>(threadPool);
    List<Future<String>> futureList = new ArrayList<Future<String>>();
    for(int i=0; i < 20; i++){
        futureList.add(completionService.submit(new Count10(i)));
    }
    for(int i=0; i < 20; i++){
        Future<String> future = completionService.take();
        System.out.println(future.get());
    }
}
 

AsynchronousGetRecordsRetrievalStrategy(KinesisDataFetcher dataFetcher, ExecutorService executorService,
        int retryGetRecordsInSeconds, Supplier<CompletionService<DataFetcherResult>> completionServiceSupplier,
        String shardId) {
    this.dataFetcher = dataFetcher;
    this.executorService = executorService;
    this.retryGetRecordsInSeconds = retryGetRecordsInSeconds;
    this.completionServiceSupplier = completionServiceSupplier;
    this.shardId = shardId;
}
 

private long executeQueries(final List<String> queries, final Context ctx) throws Exception {
    int maxThreads = KylinConfig.getInstanceFromEnv().getMigrationRuleQueryLatencyMaxThreads();
    int threadNum = Math.min(maxThreads, queries.size());
    ExecutorService threadPool = Executors.newFixedThreadPool(threadNum);
    CompletionService<Long> completionService = new ExecutorCompletionService<Long>(threadPool);
    final Authentication auth = SecurityContextHolder.getContext().getAuthentication();
    long start = System.currentTimeMillis();
    for (final String query : queries) {
        completionService.submit(new Callable<Long>() {
            @Override
            public Long call() throws Exception {
                SecurityContextHolder.getContext().setAuthentication(auth);
                SQLRequest sqlRequest = new SQLRequest();
                sqlRequest.setProject(ctx.getSrcProjectName());
                sqlRequest.setSql(query);
                SQLResponse sqlResponse = ctx.getQueryService().doQueryWithCache(sqlRequest, false);
                if (sqlResponse.getIsException()) {
                    throw new RuleValidationException(sqlResponse.getExceptionMessage());
                }
                return sqlResponse.getDuration();
            }

        });
    }
    long timeCostSum = 0L;
    for (int i = 0; i < queries.size(); ++i) {
        try {
            timeCostSum += completionService.take().get();
        } catch (InterruptedException | ExecutionException e) {
            threadPool.shutdownNow();
            throw e;
        }
    }
    long end = System.currentTimeMillis();
    logger.info("Execute" + queries.size() + " queries took " + (end - start) + " ms, query time cost sum "
            + timeCostSum + " ms.");
    return timeCostSum / queries.size();
}