Java Code Examples for java.util.concurrent.ScheduledExecutorService#awaitTermination()

The following examples show how to use java.util.concurrent.ScheduledExecutorService#awaitTermination() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public static void killTasks(final String name) {
    final ScheduledExecutorService executorService = getBackgroundExecutor(name);
    executorService.shutdownNow();
    try {
        executorService.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Log.wtf(TAG, "Failed to shut down: " + name);
    }
    if (executorService == sExecutorServiceForTests) {
        // Don't do anything to the test service.
        return;
    }
    switch (name) {
        case KEYBOARD:
            sKeyboardExecutorService = newExecutorService(KEYBOARD);
            break;
        case SPELLING:
            sSpellingExecutorService = newExecutorService(SPELLING);
            break;
        default:
            throw new IllegalArgumentException("Invalid executor: " + name);
    }
}
 
Example 2
public void stop() throws Exception {
  if (!stopped.getAndSet(true)) {
    executorPools.forEach(ScheduledExecutorService::shutdown);

    long timeoutLeftInMillis = timeoutInMillis;

    for (ScheduledExecutorService service : executorPools) {
      final long start = System.currentTimeMillis();

      if (!service.awaitTermination(timeoutLeftInMillis, TimeUnit.MILLISECONDS)) {
        return;
      }

      timeoutLeftInMillis -= (System.currentTimeMillis() - start);
    }
  }
}
 
Example 3
@Test
public void testDelay() throws InterruptedException {
    ScheduledExecutorService ses =
        Executors.newSingleThreadScheduledExecutor();

    SingletonTask st1 = new SingletonTask(ses, new Runnable() {
        @Override
        public void run() {
            ran += 1;
            time = System.nanoTime();
        }
    });
    long start = System.nanoTime();
    st1.reschedule(10, TimeUnit.MILLISECONDS);
    assertFalse("Check that task hasn't run yet", ran > 0);

    ses.shutdown();
    ses.awaitTermination(5, TimeUnit.SECONDS);

    assertEquals("Check that task ran", 1, ran);
    assertTrue("Check that time passed appropriately",
               (time - start) >= TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS));
}
 
Example 4
@Test
public void test1 () throws Exception
{
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor ();

    final Input input = new UrlConnectionInput ( executor, new URL ( "http://google.com" ), true, 100 );

    final TestListener listener = new TestListener ();

    input.addInputListener ( listener );

    input.start ();
    Thread.sleep ( 100 );
    input.stop ();
    input.dispose ();

    executor.shutdown ();
    executor.awaitTermination ( Long.MAX_VALUE, TimeUnit.MINUTES );

    dumpData ( listener );

    // TODO: test
}
 
Example 5
Source Project: tomee   File: Pool.java    License: Apache License 2.0 6 votes vote down vote up
public void stop() {
    final ScheduledFuture<?> future = this.future.getAndSet(null);
    if (future != null
            && !future.isDone() && !future.isCancelled()
            && !future.cancel(false)) {
        Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler task termination timeout expired");
    }

    final ScheduledExecutorService scheduler = this.scheduler.getAndSet(null);
    if (scheduler != null) {
        scheduler.shutdown();
        try {
            if (!scheduler.awaitTermination(10, SECONDS)) { // should last something like 0s max since we killed the task
                Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler termination timeout expired");
            }
        } catch (final InterruptedException e) {
            //Ignore
        }
    }
}
 
Example 6
Source Project: kylin   File: StandaloneExample.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Runs a number of threads which generate metrics.
 */
public static void generateMetrics(final MetricRegistry metrics, final long metricsToGenerate, final int period,
        final TimeUnit periodTimeUnit, HadoopMetrics2Reporter metrics2Reporter, int numThreads) throws Exception {
    final ScheduledExecutorService pool = Executors.newScheduledThreadPool(numThreads);
    final CountDownLatch latch = new CountDownLatch(numThreads);

    for (int i = 0; i < numThreads; i++) {
        final int id = i;
        final int halfPeriod = (period / 2);
        Runnable task = new Runnable() {
            private long executions = 0;
            final Random r = new Random();

            @Override
            public void run() {
                if (executions >= metricsToGenerate) {
                    return;
                }
                metrics.counter("foo counter thread" + id).inc();
                executions++;
                if (executions < metricsToGenerate) {
                    pool.schedule(this, period + r.nextInt(halfPeriod), periodTimeUnit);
                } else {
                    latch.countDown();
                }
            }
        };
        pool.schedule(task, period, periodTimeUnit);
    }

    while (!latch.await(2, TimeUnit.SECONDS)) {
        metrics2Reporter.printQueueDebugMessage();
    }

    pool.shutdown();
    pool.awaitTermination(5000, TimeUnit.SECONDS);
}
 
Example 7
Source Project: tinkerpop   File: GremlinExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void shouldNotShutdownExecutorServicesSuppliedToGremlinExecutor() throws Exception {
    final ScheduledExecutorService service = Executors.newScheduledThreadPool(4, testingThreadFactory);
    final GremlinExecutor gremlinExecutor = GremlinExecutor.build()
            .executorService(service)
            .scheduledExecutorService(service).create();

    gremlinExecutor.close();
    assertFalse(service.isShutdown());
    service.shutdown();
    service.awaitTermination(30000, TimeUnit.MILLISECONDS);
}
 
Example 8
Source Project: r2cloud   File: Util.java    License: Apache License 2.0 5 votes vote down vote up
public static void shutdown(ScheduledExecutorService executor, long timeoutMillis) {
	if (executor == null) {
		return;
	}
	executor.shutdownNow();
	boolean cleanlyTerminated;
	try {
		cleanlyTerminated = executor.awaitTermination(timeoutMillis, TimeUnit.MILLISECONDS);
	} catch (InterruptedException e) {
		Thread.currentThread().interrupt();
		cleanlyTerminated = executor.isTerminated();
	}
	if (!cleanlyTerminated) {
		String threadpoolName;
		if (executor instanceof ScheduledThreadPoolExecutor) {
			ThreadFactory factory = ((ScheduledThreadPoolExecutor) executor).getThreadFactory();
			if (factory instanceof NamingThreadFactory) {
				NamingThreadFactory namingFactory = (NamingThreadFactory) factory;
				threadpoolName = namingFactory.getPrefix();
			} else {
				threadpoolName = "unknown[" + factory.getClass().getSimpleName() + "]";
			}
		} else {
			threadpoolName = "unknown[" + executor.getClass().getSimpleName() + "]";
		}
		LOG.error("executor did not terminate in the specified time: {}", threadpoolName);
	}
}
 
Example 9
@Test
public void sameIP() throws InterruptedException {
    List<Throttle> throttles = Collections.singletonList(
            new Throttle("IP", new LimitRate(1, Duration.ofMillis(500L)), HttpRequest::getRemoteAddr)
    );

    ThrottlingMiddleware<HttpResponse> middleware = builder(new ThrottlingMiddleware<HttpResponse>())
            .set(ThrottlingMiddleware::setThrottles, throttles)
            .build();

    DefaultHttpRequest req = new DefaultHttpRequest();
    req.setRemoteAddr("127.0.0.1");
    MiddlewareChain<HttpRequest, HttpResponse, ?, ?> chain = new DefaultMiddlewareChain<>(new AnyPredicate<>(), null,
            (Endpoint<HttpRequest, HttpResponse>) request -> HttpResponse.of(""));
    final AtomicInteger count200 = new AtomicInteger(0);
    ScheduledExecutorService service = Executors.newScheduledThreadPool(3);
    service.scheduleAtFixedRate(() -> {
        HttpResponse res = middleware.handle(req, chain);
        if (res.getStatus() == 200) {
            count200.addAndGet(1);
        }
    }, 0, 250, TimeUnit.MILLISECONDS);

    service.schedule(service::shutdown, 4, TimeUnit.SECONDS);
    service.awaitTermination(5, TimeUnit.SECONDS);

    assertThat(count200.get()).isGreaterThan(3);
}
 
Example 10
/**
 * Test to make sure we only execute the task once
 * no matter how many times we schedule it. 
 */
public void testExecuteOnlyOnce() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
 
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  ex.submit(new Callable() {

    public Object call() throws Exception {
      latch.await();
      return null;
    }
  });
  
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  for(int i = 0; i < 50; i++) {
    decorator.schedule(increment, 0, TimeUnit.SECONDS);
  }
  
  assertEquals(0, counter.get());
  latch.countDown();
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  assertEquals(1, counter.get());
  assertEquals(49, listener.getDropCount());
}
 
Example 11
/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 
Example 12
/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 
Example 13
@Test
public void testPutHeadChunkRunnableFileGrowth() throws Exception {
    File file = File.createTempFile("test", "");

    try (final FileOutputStream outputStream = new FileOutputStream(file)) {
        outputStream.write(new byte[]{0x65});

        UUID transferId = UUID.randomUUID();
        BlobTransferTarget blobTransferTarget = mock(BlobTransferTarget.class);
        TransportService transportService = mock(TransportService.class);
        DiscoveryNode discoveryNode = mock(DiscoveryNode.class);
        DigestBlob digestBlob = mock(DigestBlob.class);
        when(digestBlob.file()).thenReturn(file);

        ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory("blob-head"));
        try {
            scheduledExecutor.schedule(new Runnable() {
                @Override
                public void run() {
                    try {
                        outputStream.write(new byte[]{0x66, 0x67, 0x68, 0x69});
                    } catch (IOException ex) {
                        //pass
                    }
                }
            }, 800, TimeUnit.MILLISECONDS);
            PutHeadChunkRunnable runnable = new PutHeadChunkRunnable(
                digestBlob, 5, transportService, blobTransferTarget, discoveryNode, transferId
            );

            @SuppressWarnings("unchecked")
            TransportFuture<TransportResponse.Empty> result = mock(TransportFuture.class);

            when(transportService.submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            )).thenReturn(result);

            runnable.run();

            verify(blobTransferTarget).putHeadChunkTransferFinished(transferId);
            verify(transportService, times(2)).submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            );
        } finally {
            scheduledExecutor.awaitTermination(1, TimeUnit.SECONDS);
            scheduledExecutor.shutdownNow();
        }
    }
}
 
Example 14
public void executeRequest(final AutoScalingGroup autoScalingGroup, AwsAutoScalingDeployUtils awsDeployUtils, InstanceStatus instanceStatus) {
    final AtomicInteger waitFor = new AtomicInteger(1);
    final AtomicBoolean inService = new AtomicBoolean(false);
    final AtomicBoolean found = new AtomicBoolean(false);


    log.info("Waiting for new instance in asGroup to come in service...");
    ScheduledExecutorService exec = Executors.newSingleThreadScheduledExecutor();

    exec.scheduleAtFixedRate(() -> {
        log.debug("existing instances : " + Arrays.toString(autoScalingGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));
        AutoScalingGroup updatedGroup = awsDeployUtils.getAutoScalingGroup();
        log.debug("Updated instances : " + Arrays.toString(updatedGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));

        if (updatedGroup.getInstances().equals(autoScalingGroup.getInstances())) {
            log.info("no new instance found in auto scaling group.");
        }

        if (newInstance == null) {
            newInstance = findNewInstance(autoScalingGroup, updatedGroup);
            if (newInstance != null && !found.get()) {
                found.set(true);
                log.info("Found new instance with id " + newInstance.getInstanceId());
            }
        }
        if (newInstance != null && instanceStatus.inService(newInstance)) {
            waitFor.decrementAndGet();
        }
    }, 30, 30, TimeUnit.SECONDS);

    try {
        while (waitFor.intValue() > 0 && System.currentTimeMillis() <= timeout) {
            Thread.sleep(3000);
        }
        log.info("Shutting down executor");
        exec.shutdown();
        log.info("awaiting termination of executor");
        exec.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        log.error(e.getMessage());
        Thread.currentThread().interrupt();
        inService.get();
    } catch (Exception t) {
        log.error("Throwable: ", t);
    }
}
 
Example 15
Source Project: DDMQ   File: Utils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * 检查消费
 *
 * @param nameSvr
 * @param address
 * @throws MQClientException
 * @throws NoSuchFieldException
 * @throws SecurityException
 * @throws IllegalArgumentException
 * @throws IllegalAccessException
 * @throws InterruptedException
 * @throws RemotingException
 * @throws MQBrokerException
 */
public static long checkReceive(String cluster, String nameSvr, String address)
        throws MQClientException, NoSuchFieldException, SecurityException, IllegalArgumentException,
        IllegalAccessException, InterruptedException, RemotingException, MQBrokerException {

    DefaultMQPullConsumer consumer = getReceiveCheckConsumer(nameSvr, cluster, address);
    Field f1 = DefaultMQPullConsumerImpl.class.getDeclaredField("mQClientFactory");
    f1.setAccessible(true);

    MQClientInstance instance = (MQClientInstance) f1.get(consumer.getDefaultMQPullConsumerImpl());

    Field f = MQClientInstance.class.getDeclaredField("brokerAddrTable");
    f.setAccessible(true);

    Field f2 = MQClientInstance.class.getDeclaredField("scheduledExecutorService");
    f2.setAccessible(true);

    ScheduledExecutorService service = (ScheduledExecutorService) f2.get(instance);
    service.shutdown();
    service.awaitTermination(1000, TimeUnit.SECONDS);

    ConcurrentHashMap<String, HashMap<Long, String>> map = (ConcurrentHashMap<String, HashMap<Long, String>>) f.get(instance);
    HashMap<Long, String> addresses = new HashMap<>();
    addresses.put(0L, address);
    map.put("rmqmonitor_" + address, addresses);

    MessageQueue queue = new MessageQueue("SELF_TEST_TOPIC", "rmqmonitor_" + address, 0);

    boolean pullOk = false;
    long maxOffset = -1;
    for (int i = 0; i < 2; ++i) {
        try {
            maxOffset = consumer.getDefaultMQPullConsumerImpl().maxOffset(queue);
            PullResult result = consumer.pull(queue, "*", maxOffset > 100 ? maxOffset - 10 : 0, 1);
            if (result.getPullStatus() == PullStatus.FOUND) {
                pullOk = true;
                break;
            } else if(result.getPullStatus() == PullStatus.NO_NEW_MSG) {
                checkSend(cluster, nameSvr, address);
                continue;
            }

            logger.warn("pull result failed, PullResult={}, cluster={}, namesvr={}, address={}", result, cluster, nameSvr, address);
        } catch (Throwable e) {
            logger.error("pull exception, cluster={}, namesvr={}, address={}", cluster, nameSvr, address, e);
        }
        Thread.sleep(1000);
    }
    if (!pullOk) {
        logger.error(String.format("[AlarmPullErr] cluster=%s, broker=%s", cluster, address));
    } else {
        logger.info("AlarmPullCheck cluster={}, broker={}", cluster, address);
    }
    return maxOffset;
}
 
Example 16
Source Project: armeria   File: Server.java    License: Apache License 2.0 4 votes vote down vote up
private void finishDoStop(CompletableFuture<Void> future) {
    serverChannels.clear();

    if (config.shutdownBlockingTaskExecutorOnStop()) {
        final ScheduledExecutorService executor;
        final ScheduledExecutorService blockingTaskExecutor = config.blockingTaskExecutor();
        if (blockingTaskExecutor instanceof UnstoppableScheduledExecutorService) {
            executor =
                    ((UnstoppableScheduledExecutorService) blockingTaskExecutor).getExecutorService();
        } else {
            executor = blockingTaskExecutor;
        }

        try {
            executor.shutdown();
            while (!executor.isTerminated()) {
                try {
                    executor.awaitTermination(1, TimeUnit.DAYS);
                } catch (InterruptedException ignore) {
                    // Do nothing.
                }
            }
        } catch (Exception e) {
            logger.warn("Failed to shutdown the blockingTaskExecutor: {}", executor, e);
        }
    }

    final Builder<AccessLogWriter> builder = ImmutableSet.builder();
    config.virtualHosts()
          .stream()
          .filter(VirtualHost::shutdownAccessLogWriterOnStop)
          .forEach(virtualHost -> builder.add(virtualHost.accessLogWriter()));
    config.serviceConfigs()
          .stream()
          .filter(ServiceConfig::shutdownAccessLogWriterOnStop)
          .forEach(serviceConfig -> builder.add(serviceConfig.accessLogWriter()));

    final Set<AccessLogWriter> writers = builder.build();
    final List<CompletableFuture<Void>> completionFutures = new ArrayList<>(writers.size());
    writers.forEach(accessLogWriter -> {
        final CompletableFuture<Void> shutdownFuture = accessLogWriter.shutdown();
        shutdownFuture.exceptionally(cause -> {
            logger.warn("Failed to close the {}:", AccessLogWriter.class.getSimpleName(), cause);
            return null;
        });
        completionFutures.add(shutdownFuture);
    });

    CompletableFutures.successfulAsList(completionFutures, cause -> null)
                      .thenRunAsync(() -> future.complete(null), config.startStopExecutor());
}
 
Example 17
Source Project: fahrschein   File: Main.java    License: Apache License 2.0 4 votes vote down vote up
private static void multiInstanceListen(ObjectMapper objectMapper, Listener<SalesOrderPlaced> listener) throws IOException {
    final HikariConfig hikariConfig = new HikariConfig();
    hikariConfig.setJdbcUrl(JDBC_URL);
    hikariConfig.setUsername(JDBC_USERNAME);
    hikariConfig.setPassword(JDBC_PASSWORD);

    final DataSource dataSource = new HikariDataSource(hikariConfig);

    final ZignAccessTokenProvider accessTokenProvider = new ZignAccessTokenProvider();

    final AtomicInteger name = new AtomicInteger();
    final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(16);

    for (int i = 0; i < 12; i++) {
        final String instanceName = "consumer-" + name.getAndIncrement();
        final JdbcPartitionManager partitionManager = new JdbcPartitionManager(dataSource, "fahrschein-demo");
        final JdbcCursorManager cursorManager = new JdbcCursorManager(dataSource, "fahrschein-demo");

        final NakadiClient nakadiClient = NakadiClient.builder(NAKADI_URI)
                .withAccessTokenProvider(accessTokenProvider)
                .withCursorManager(cursorManager)
                .build();

        final List<Partition> partitions = nakadiClient.getPartitions(SALES_ORDER_SERVICE_ORDER_PLACED);

        final IORunnable instance = () -> {

            final IORunnable runnable = () -> {
                final Optional<Lock> optionalLock = partitionManager.lockPartitions(SALES_ORDER_SERVICE_ORDER_PLACED, partitions, instanceName);

                if (optionalLock.isPresent()) {
                    final Lock lock = optionalLock.get();
                    try {
                        nakadiClient.stream(SALES_ORDER_SERVICE_ORDER_PLACED)
                                .withLock(lock)
                                .withObjectMapper(objectMapper)
                                .withStreamParameters(new StreamParameters().withStreamLimit(10))
                                .withBackoffStrategy(new NoBackoffStrategy())
                                .listen(SalesOrderPlaced.class, listener);
                    } finally {
                        partitionManager.unlockPartitions(lock);
                    }
                }
            };

            scheduledExecutorService.scheduleWithFixedDelay(runnable.unchecked(), 0, 1, TimeUnit.SECONDS);
        };
        scheduledExecutorService.submit(instance.unchecked());
    }

    try {
        Thread.sleep(60L*1000);
        scheduledExecutorService.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}
 
Example 18
@Test
public void testAsyncDirFinderShuffledDirsCreatedPeriodically() throws Exception {
  String testDataDir = "target" + File.separatorChar + UUID.randomUUID().toString();
  Files.createDirectories(Paths.get(testDataDir));
  final Collection<Path> paths = Arrays.asList(
      Paths.get(testDataDir + File.separatorChar + "d1"),
      Paths.get(testDataDir + File.separatorChar + "d2"),
      Paths.get(testDataDir + File.separatorChar + "d3"),
      Paths.get(testDataDir + File.separatorChar + "d4"),
      Paths.get(testDataDir + File.separatorChar + "d5"),
      Paths.get(testDataDir + File.separatorChar + "d6"),
      Paths.get(testDataDir + File.separatorChar + "d7"),
      Paths.get(testDataDir + File.separatorChar + "d8"),

      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d11"),
      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d12"),

      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d21" + File.separatorChar + "d211"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d22" + File.separatorChar + "d221"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d23" + File.separatorChar + "d231"),

      Paths.get(testDataDir + File.separatorChar + "d3" + File.separatorChar + "d31"),

      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111" + File.separatorChar + "d411111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112" + File.separatorChar + "d411111")

  );
  DirectoryPathCreationWatcher watcher = new DirectoryPathCreationWatcher(paths, 1);

  ScheduledExecutorService directoryCreationExecutor = new SafeScheduledExecutorService(1, "Directory Creator");
  DirectoryCreationWorker worker = new DirectoryCreationWorker(paths, true);
  directoryCreationExecutor.scheduleAtFixedRate(worker, 0, 100, TimeUnit.MILLISECONDS);

  Set<Path> foundPaths = new HashSet<Path>();
  while (!worker.getRemainingPathsToCreate().isEmpty()) {
    Set<Path> foundPathsTillNow = watcher.find();
    foundPaths.addAll(foundPathsTillNow);
    Thread.sleep(100);
  }

  directoryCreationExecutor.shutdownNow();
  directoryCreationExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);

  Thread.sleep(paths.size() * 100 + 1000);

  //Just to find the last set of detected directories.
  foundPaths.addAll(watcher.find());

  checkCorrectness(paths, foundPaths, (ScheduledExecutorService)Whitebox.getInternalState(watcher, "executor"));
}
 
Example 19
/**
 * Test that a streaming task will work correctly when an extractor is continuously producing records and encounters
 * an error in the writer.
 *
 * The task should exit in a failed state.
 *
 * No converters
 * Identity fork
 * One writer
 * @throws Exception
 */
@Test
public void testContinuousTaskError()
    throws Exception {

  for (Boolean taskExecutionSync: new Boolean[]{true, false}) {
    ArrayList<Object> recordCollector = new ArrayList<>(100);
    long perRecordExtractLatencyMillis = 1000; // 1 second per record

    ContinuousExtractor continuousExtractor = new ContinuousExtractor(perRecordExtractLatencyMillis);

    TaskContext mockTaskContext = getMockTaskContext(recordCollector, continuousExtractor, taskExecutionSync, 5);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent());

    ScheduledExecutorService taskRunner = new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(log)));

    taskRunner.execute(task);

    // Let the task run for 10 seconds
    int sleepIterations = 10;
    int currentIteration = 0;

    while (currentIteration < sleepIterations) {
      Thread.sleep(1000);
      currentIteration++;
      Map<String, CheckpointableWatermark> externalWatermarkStorage = mockTaskContext.getWatermarkStorage()
          .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"));
      if (!externalWatermarkStorage.isEmpty()) {
        for (CheckpointableWatermark watermark : externalWatermarkStorage.values()) {
          log.info("Observed committed watermark: {}", watermark);
        }
        log.info("Task progress: {}", task.getProgress());
        // Ensure that watermarks seem reasonable at each step
        Assert.assertTrue(continuousExtractor.validateWatermarks(false, externalWatermarkStorage));
      }
    }

    boolean success = task.awaitShutdown(30000);
    Assert.assertTrue(success, "Task should shutdown in 30 seconds");
    log.info("Task done waiting to shutdown {}", success);

    // Shutdown on error, so don't check for the exact watermark
    Assert.assertTrue(continuousExtractor.validateWatermarks(false, mockTaskContext.getWatermarkStorage()
        .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"))));

    task.commit();

    // Task should be in failed state from extractor error
    Assert.assertEquals(mockTaskContext.getTaskState().getWorkingState(), WorkUnitState.WorkingState.FAILED);
    // Shutdown the executor
    taskRunner.shutdown();
    taskRunner.awaitTermination(100, TimeUnit.MILLISECONDS);
  }
}
 
Example 20
Source Project: nifi   File: StandardFlowService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void stop(final boolean force) {
    writeLock.lock();
    try {

        if (!isRunning()) {
            return;
        }

        running.set(false);

        if (clusterCoordinator != null) {
            final Thread shutdownClusterCoordinator = new Thread(clusterCoordinator::shutdown);

            shutdownClusterCoordinator.setDaemon(true);
            shutdownClusterCoordinator.setName("Shutdown Cluster Coordinator");
            shutdownClusterCoordinator.start();
        }

        if (!controller.isTerminated()) {
            controller.shutdown(force);
        }

        if (configuredForClustering && senderListener != null) {
            try {
                senderListener.stop();
            } catch (final IOException ioe) {
                logger.warn("Protocol sender/listener did not stop gracefully due to: " + ioe);
            }
        }

        final ScheduledExecutorService executorService = executor.get();
        if (executorService != null) {
            if (force) {
                executorService.shutdownNow();
            } else {
                executorService.shutdown();
            }

            boolean graceful;
            try {
                graceful = executorService.awaitTermination(gracefulShutdownSeconds, TimeUnit.SECONDS);
            } catch (final InterruptedException e) {
                graceful = false;
            }

            if (!graceful) {
                logger.warn("Scheduling service did not gracefully shutdown within configured " + gracefulShutdownSeconds + " second window");
            }
        }
    } finally {
        writeLock.unlock();
    }
}