Java Code Examples for java.util.concurrent.ScheduledExecutorService#awaitTermination()

The following examples show how to use java.util.concurrent.ScheduledExecutorService#awaitTermination() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SingularityManagedScheduledExecutorServiceFactory.java    From Singularity with Apache License 2.0 6 votes vote down vote up
public void stop() throws Exception {
  if (!stopped.getAndSet(true)) {
    executorPools.forEach(ScheduledExecutorService::shutdown);

    long timeoutLeftInMillis = timeoutInMillis;

    for (ScheduledExecutorService service : executorPools) {
      final long start = System.currentTimeMillis();

      if (!service.awaitTermination(timeoutLeftInMillis, TimeUnit.MILLISECONDS)) {
        return;
      }

      timeoutLeftInMillis -= (System.currentTimeMillis() - start);
    }
  }
}
 
Example 2
Source File: Pool.java    From tomee with Apache License 2.0 6 votes vote down vote up
public void stop() {
    final ScheduledFuture<?> future = this.future.getAndSet(null);
    if (future != null
            && !future.isDone() && !future.isCancelled()
            && !future.cancel(false)) {
        Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler task termination timeout expired");
    }

    final ScheduledExecutorService scheduler = this.scheduler.getAndSet(null);
    if (scheduler != null) {
        scheduler.shutdown();
        try {
            if (!scheduler.awaitTermination(10, SECONDS)) { // should last something like 0s max since we killed the task
                Logger.getLogger(Pool.class.getName()).log(Level.WARNING, "Pool scheduler termination timeout expired");
            }
        } catch (final InterruptedException e) {
            //Ignore
        }
    }
}
 
Example 3
Source File: UrlConnectionTest.java    From neoscada with Eclipse Public License 1.0 6 votes vote down vote up
@Test
public void test1 () throws Exception
{
    final ScheduledExecutorService executor = Executors.newSingleThreadScheduledExecutor ();

    final Input input = new UrlConnectionInput ( executor, new URL ( "http://google.com" ), true, 100 );

    final TestListener listener = new TestListener ();

    input.addInputListener ( listener );

    input.start ();
    Thread.sleep ( 100 );
    input.stop ();
    input.dispose ();

    executor.shutdown ();
    executor.awaitTermination ( Long.MAX_VALUE, TimeUnit.MINUTES );

    dumpData ( listener );

    // TODO: test
}
 
Example 4
Source File: SingletonTaskTest.java    From floodlight_with_topoguard with Apache License 2.0 6 votes vote down vote up
@Test
public void testDelay() throws InterruptedException {
    ScheduledExecutorService ses =
        Executors.newSingleThreadScheduledExecutor();

    SingletonTask st1 = new SingletonTask(ses, new Runnable() {
        @Override
        public void run() {
            ran += 1;
            time = System.nanoTime();
        }
    });
    long start = System.nanoTime();
    st1.reschedule(10, TimeUnit.MILLISECONDS);
    assertFalse("Check that task hasn't run yet", ran > 0);

    ses.shutdown();
    ses.awaitTermination(5, TimeUnit.SECONDS);

    assertEquals("Check that task ran", 1, ran);
    assertTrue("Check that time passed appropriately",
               (time - start) >= TimeUnit.NANOSECONDS.convert(10, TimeUnit.MILLISECONDS));
}
 
Example 5
Source File: ExecutorUtils.java    From openboard with GNU General Public License v3.0 6 votes vote down vote up
public static void killTasks(final String name) {
    final ScheduledExecutorService executorService = getBackgroundExecutor(name);
    executorService.shutdownNow();
    try {
        executorService.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Log.wtf(TAG, "Failed to shut down: " + name);
    }
    if (executorService == sExecutorServiceForTests) {
        // Don't do anything to the test service.
        return;
    }
    switch (name) {
        case KEYBOARD:
            sKeyboardExecutorService = newExecutorService(KEYBOARD);
            break;
        case SPELLING:
            sSpellingExecutorService = newExecutorService(SPELLING);
            break;
        default:
            throw new IllegalArgumentException("Invalid executor: " + name);
    }
}
 
Example 6
Source File: GremlinExecutorTest.java    From tinkerpop with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldNotShutdownExecutorServicesSuppliedToGremlinExecutor() throws Exception {
    final ScheduledExecutorService service = Executors.newScheduledThreadPool(4, testingThreadFactory);
    final GremlinExecutor gremlinExecutor = GremlinExecutor.build()
            .executorService(service)
            .scheduledExecutorService(service).create();

    gremlinExecutor.close();
    assertFalse(service.isShutdown());
    service.shutdown();
    service.awaitTermination(30000, TimeUnit.MILLISECONDS);
}
 
Example 7
Source File: StandaloneExample.java    From kylin with Apache License 2.0 5 votes vote down vote up
/**
 * Runs a number of threads which generate metrics.
 */
public static void generateMetrics(final MetricRegistry metrics, final long metricsToGenerate, final int period,
        final TimeUnit periodTimeUnit, HadoopMetrics2Reporter metrics2Reporter, int numThreads) throws Exception {
    final ScheduledExecutorService pool = Executors.newScheduledThreadPool(numThreads);
    final CountDownLatch latch = new CountDownLatch(numThreads);

    for (int i = 0; i < numThreads; i++) {
        final int id = i;
        final int halfPeriod = (period / 2);
        Runnable task = new Runnable() {
            private long executions = 0;
            final Random r = new Random();

            @Override
            public void run() {
                if (executions >= metricsToGenerate) {
                    return;
                }
                metrics.counter("foo counter thread" + id).inc();
                executions++;
                if (executions < metricsToGenerate) {
                    pool.schedule(this, period + r.nextInt(halfPeriod), periodTimeUnit);
                } else {
                    latch.countDown();
                }
            }
        };
        pool.schedule(task, period, periodTimeUnit);
    }

    while (!latch.await(2, TimeUnit.SECONDS)) {
        metrics2Reporter.printQueueDebugMessage();
    }

    pool.shutdown();
    pool.awaitTermination(5000, TimeUnit.SECONDS);
}
 
Example 8
Source File: Util.java    From r2cloud with Apache License 2.0 5 votes vote down vote up
public static void shutdown(ScheduledExecutorService executor, long timeoutMillis) {
	if (executor == null) {
		return;
	}
	executor.shutdownNow();
	boolean cleanlyTerminated;
	try {
		cleanlyTerminated = executor.awaitTermination(timeoutMillis, TimeUnit.MILLISECONDS);
	} catch (InterruptedException e) {
		Thread.currentThread().interrupt();
		cleanlyTerminated = executor.isTerminated();
	}
	if (!cleanlyTerminated) {
		String threadpoolName;
		if (executor instanceof ScheduledThreadPoolExecutor) {
			ThreadFactory factory = ((ScheduledThreadPoolExecutor) executor).getThreadFactory();
			if (factory instanceof NamingThreadFactory) {
				NamingThreadFactory namingFactory = (NamingThreadFactory) factory;
				threadpoolName = namingFactory.getPrefix();
			} else {
				threadpoolName = "unknown[" + factory.getClass().getSimpleName() + "]";
			}
		} else {
			threadpoolName = "unknown[" + executor.getClass().getSimpleName() + "]";
		}
		LOG.error("executor did not terminate in the specified time: {}", threadpoolName);
	}
}
 
Example 9
Source File: ThrottlingMiddlewareTest.java    From enkan with Eclipse Public License 1.0 5 votes vote down vote up
@Test
public void sameIP() throws InterruptedException {
    List<Throttle> throttles = Collections.singletonList(
            new Throttle("IP", new LimitRate(1, Duration.ofMillis(500L)), HttpRequest::getRemoteAddr)
    );

    ThrottlingMiddleware<HttpResponse> middleware = builder(new ThrottlingMiddleware<HttpResponse>())
            .set(ThrottlingMiddleware::setThrottles, throttles)
            .build();

    DefaultHttpRequest req = new DefaultHttpRequest();
    req.setRemoteAddr("127.0.0.1");
    MiddlewareChain<HttpRequest, HttpResponse, ?, ?> chain = new DefaultMiddlewareChain<>(new AnyPredicate<>(), null,
            (Endpoint<HttpRequest, HttpResponse>) request -> HttpResponse.of(""));
    final AtomicInteger count200 = new AtomicInteger(0);
    ScheduledExecutorService service = Executors.newScheduledThreadPool(3);
    service.scheduleAtFixedRate(() -> {
        HttpResponse res = middleware.handle(req, chain);
        if (res.getStatus() == 200) {
            count200.addAndGet(1);
        }
    }, 0, 250, TimeUnit.MILLISECONDS);

    service.schedule(service::shutdown, 4, TimeUnit.SECONDS);
    service.awaitTermination(5, TimeUnit.SECONDS);

    assertThat(count200.get()).isGreaterThan(3);
}
 
Example 10
Source File: OneTaskOnlyDecoratorJUnitTest.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Test to make sure we only execute the task once
 * no matter how many times we schedule it. 
 */
public void testExecuteOnlyOnce() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
 
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  ex.submit(new Callable() {

    public Object call() throws Exception {
      latch.await();
      return null;
    }
  });
  
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  for(int i = 0; i < 50; i++) {
    decorator.schedule(increment, 0, TimeUnit.SECONDS);
  }
  
  assertEquals(0, counter.get());
  latch.countDown();
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  assertEquals(1, counter.get());
  assertEquals(49, listener.getDropCount());
}
 
Example 11
Source File: OneTaskOnlyDecoratorJUnitTest.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 
Example 12
Source File: OneTaskOnlyDecoratorJUnitTest.java    From gemfirexd-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Test to make sure we reschedule the task for execution 
 * if the new requested execution is earlier than the previous one
 */
public void testRescheduleForEarlierTime() throws InterruptedException {
  ScheduledExecutorService ex = Executors.newScheduledThreadPool(1);
  MyConflationListener listener = new MyConflationListener();
  OneTaskOnlyExecutor decorator = new OneTaskOnlyExecutor(ex, listener);
  
  final CountDownLatch latch = new CountDownLatch(1);
  final AtomicInteger counter = new AtomicInteger();
  
  Runnable increment = new Runnable() {

    public void run() {
      counter.incrementAndGet();
    }
  };
  
  decorator.schedule(increment, 120, TimeUnit.SECONDS);
  decorator.schedule(increment, 10, TimeUnit.MILLISECONDS);
  
  long start = System.nanoTime();
  
  ex.shutdown();
  ex.awaitTermination(60, TimeUnit.SECONDS);
  long elapsed = System.nanoTime() - start;
  assertEquals(1, counter.get());
  assertEquals(1, listener.getDropCount());
  assertTrue(elapsed < TimeUnit.SECONDS.toNanos(120));
}
 
Example 13
Source File: WaitForInstanceRequestExecutor.java    From vertx-deploy-tools with Apache License 2.0 4 votes vote down vote up
public void executeRequest(final AutoScalingGroup autoScalingGroup, AwsAutoScalingDeployUtils awsDeployUtils, InstanceStatus instanceStatus) {
    final AtomicInteger waitFor = new AtomicInteger(1);
    final AtomicBoolean inService = new AtomicBoolean(false);
    final AtomicBoolean found = new AtomicBoolean(false);


    log.info("Waiting for new instance in asGroup to come in service...");
    ScheduledExecutorService exec = Executors.newSingleThreadScheduledExecutor();

    exec.scheduleAtFixedRate(() -> {
        log.debug("existing instances : " + Arrays.toString(autoScalingGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));
        AutoScalingGroup updatedGroup = awsDeployUtils.getAutoScalingGroup();
        log.debug("Updated instances : " + Arrays.toString(updatedGroup.getInstances().stream().map(Instance::getInstanceId).toArray()));

        if (updatedGroup.getInstances().equals(autoScalingGroup.getInstances())) {
            log.info("no new instance found in auto scaling group.");
        }

        if (newInstance == null) {
            newInstance = findNewInstance(autoScalingGroup, updatedGroup);
            if (newInstance != null && !found.get()) {
                found.set(true);
                log.info("Found new instance with id " + newInstance.getInstanceId());
            }
        }
        if (newInstance != null && instanceStatus.inService(newInstance)) {
            waitFor.decrementAndGet();
        }
    }, 30, 30, TimeUnit.SECONDS);

    try {
        while (waitFor.intValue() > 0 && System.currentTimeMillis() <= timeout) {
            Thread.sleep(3000);
        }
        log.info("Shutting down executor");
        exec.shutdown();
        log.info("awaiting termination of executor");
        exec.awaitTermination(5, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        log.error(e.getMessage());
        Thread.currentThread().interrupt();
        inService.get();
    } catch (Exception t) {
        log.error("Throwable: ", t);
    }
}
 
Example 14
Source File: Utils.java    From DDMQ with Apache License 2.0 4 votes vote down vote up
/**
 * 检查消费
 *
 * @param nameSvr
 * @param address
 * @throws MQClientException
 * @throws NoSuchFieldException
 * @throws SecurityException
 * @throws IllegalArgumentException
 * @throws IllegalAccessException
 * @throws InterruptedException
 * @throws RemotingException
 * @throws MQBrokerException
 */
public static long checkReceive(String cluster, String nameSvr, String address)
        throws MQClientException, NoSuchFieldException, SecurityException, IllegalArgumentException,
        IllegalAccessException, InterruptedException, RemotingException, MQBrokerException {

    DefaultMQPullConsumer consumer = getReceiveCheckConsumer(nameSvr, cluster, address);
    Field f1 = DefaultMQPullConsumerImpl.class.getDeclaredField("mQClientFactory");
    f1.setAccessible(true);

    MQClientInstance instance = (MQClientInstance) f1.get(consumer.getDefaultMQPullConsumerImpl());

    Field f = MQClientInstance.class.getDeclaredField("brokerAddrTable");
    f.setAccessible(true);

    Field f2 = MQClientInstance.class.getDeclaredField("scheduledExecutorService");
    f2.setAccessible(true);

    ScheduledExecutorService service = (ScheduledExecutorService) f2.get(instance);
    service.shutdown();
    service.awaitTermination(1000, TimeUnit.SECONDS);

    ConcurrentHashMap<String, HashMap<Long, String>> map = (ConcurrentHashMap<String, HashMap<Long, String>>) f.get(instance);
    HashMap<Long, String> addresses = new HashMap<>();
    addresses.put(0L, address);
    map.put("rmqmonitor_" + address, addresses);

    MessageQueue queue = new MessageQueue("SELF_TEST_TOPIC", "rmqmonitor_" + address, 0);

    boolean pullOk = false;
    long maxOffset = -1;
    for (int i = 0; i < 2; ++i) {
        try {
            maxOffset = consumer.getDefaultMQPullConsumerImpl().maxOffset(queue);
            PullResult result = consumer.pull(queue, "*", maxOffset > 100 ? maxOffset - 10 : 0, 1);
            if (result.getPullStatus() == PullStatus.FOUND) {
                pullOk = true;
                break;
            } else if(result.getPullStatus() == PullStatus.NO_NEW_MSG) {
                checkSend(cluster, nameSvr, address);
                continue;
            }

            logger.warn("pull result failed, PullResult={}, cluster={}, namesvr={}, address={}", result, cluster, nameSvr, address);
        } catch (Throwable e) {
            logger.error("pull exception, cluster={}, namesvr={}, address={}", cluster, nameSvr, address, e);
        }
        Thread.sleep(1000);
    }
    if (!pullOk) {
        logger.error(String.format("[AlarmPullErr] cluster=%s, broker=%s", cluster, address));
    } else {
        logger.info("AlarmPullCheck cluster={}, broker={}", cluster, address);
    }
    return maxOffset;
}
 
Example 15
Source File: Server.java    From armeria with Apache License 2.0 4 votes vote down vote up
private void finishDoStop(CompletableFuture<Void> future) {
    serverChannels.clear();

    if (config.shutdownBlockingTaskExecutorOnStop()) {
        final ScheduledExecutorService executor;
        final ScheduledExecutorService blockingTaskExecutor = config.blockingTaskExecutor();
        if (blockingTaskExecutor instanceof UnstoppableScheduledExecutorService) {
            executor =
                    ((UnstoppableScheduledExecutorService) blockingTaskExecutor).getExecutorService();
        } else {
            executor = blockingTaskExecutor;
        }

        try {
            executor.shutdown();
            while (!executor.isTerminated()) {
                try {
                    executor.awaitTermination(1, TimeUnit.DAYS);
                } catch (InterruptedException ignore) {
                    // Do nothing.
                }
            }
        } catch (Exception e) {
            logger.warn("Failed to shutdown the blockingTaskExecutor: {}", executor, e);
        }
    }

    final Builder<AccessLogWriter> builder = ImmutableSet.builder();
    config.virtualHosts()
          .stream()
          .filter(VirtualHost::shutdownAccessLogWriterOnStop)
          .forEach(virtualHost -> builder.add(virtualHost.accessLogWriter()));
    config.serviceConfigs()
          .stream()
          .filter(ServiceConfig::shutdownAccessLogWriterOnStop)
          .forEach(serviceConfig -> builder.add(serviceConfig.accessLogWriter()));

    final Set<AccessLogWriter> writers = builder.build();
    final List<CompletableFuture<Void>> completionFutures = new ArrayList<>(writers.size());
    writers.forEach(accessLogWriter -> {
        final CompletableFuture<Void> shutdownFuture = accessLogWriter.shutdown();
        shutdownFuture.exceptionally(cause -> {
            logger.warn("Failed to close the {}:", AccessLogWriter.class.getSimpleName(), cause);
            return null;
        });
        completionFutures.add(shutdownFuture);
    });

    CompletableFutures.successfulAsList(completionFutures, cause -> null)
                      .thenRunAsync(() -> future.complete(null), config.startStopExecutor());
}
 
Example 16
Source File: Main.java    From fahrschein with Apache License 2.0 4 votes vote down vote up
private static void multiInstanceListen(ObjectMapper objectMapper, Listener<SalesOrderPlaced> listener) throws IOException {
    final HikariConfig hikariConfig = new HikariConfig();
    hikariConfig.setJdbcUrl(JDBC_URL);
    hikariConfig.setUsername(JDBC_USERNAME);
    hikariConfig.setPassword(JDBC_PASSWORD);

    final DataSource dataSource = new HikariDataSource(hikariConfig);

    final ZignAccessTokenProvider accessTokenProvider = new ZignAccessTokenProvider();

    final AtomicInteger name = new AtomicInteger();
    final ScheduledExecutorService scheduledExecutorService = Executors.newScheduledThreadPool(16);

    for (int i = 0; i < 12; i++) {
        final String instanceName = "consumer-" + name.getAndIncrement();
        final JdbcPartitionManager partitionManager = new JdbcPartitionManager(dataSource, "fahrschein-demo");
        final JdbcCursorManager cursorManager = new JdbcCursorManager(dataSource, "fahrschein-demo");

        final NakadiClient nakadiClient = NakadiClient.builder(NAKADI_URI)
                .withAccessTokenProvider(accessTokenProvider)
                .withCursorManager(cursorManager)
                .build();

        final List<Partition> partitions = nakadiClient.getPartitions(SALES_ORDER_SERVICE_ORDER_PLACED);

        final IORunnable instance = () -> {

            final IORunnable runnable = () -> {
                final Optional<Lock> optionalLock = partitionManager.lockPartitions(SALES_ORDER_SERVICE_ORDER_PLACED, partitions, instanceName);

                if (optionalLock.isPresent()) {
                    final Lock lock = optionalLock.get();
                    try {
                        nakadiClient.stream(SALES_ORDER_SERVICE_ORDER_PLACED)
                                .withLock(lock)
                                .withObjectMapper(objectMapper)
                                .withStreamParameters(new StreamParameters().withStreamLimit(10))
                                .withBackoffStrategy(new NoBackoffStrategy())
                                .listen(SalesOrderPlaced.class, listener);
                    } finally {
                        partitionManager.unlockPartitions(lock);
                    }
                }
            };

            scheduledExecutorService.scheduleWithFixedDelay(runnable.unchecked(), 0, 1, TimeUnit.SECONDS);
        };
        scheduledExecutorService.submit(instance.unchecked());
    }

    try {
        Thread.sleep(60L*1000);
        scheduledExecutorService.awaitTermination(Long.MAX_VALUE, TimeUnit.SECONDS);
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    }
}
 
Example 17
Source File: BlobHeadRequestHandlerTests.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testPutHeadChunkRunnableFileGrowth() throws Exception {
    File file = File.createTempFile("test", "");

    try (final FileOutputStream outputStream = new FileOutputStream(file)) {
        outputStream.write(new byte[]{0x65});

        UUID transferId = UUID.randomUUID();
        BlobTransferTarget blobTransferTarget = mock(BlobTransferTarget.class);
        TransportService transportService = mock(TransportService.class);
        DiscoveryNode discoveryNode = mock(DiscoveryNode.class);
        DigestBlob digestBlob = mock(DigestBlob.class);
        when(digestBlob.file()).thenReturn(file);

        ScheduledExecutorService scheduledExecutor = Executors.newSingleThreadScheduledExecutor(EsExecutors.daemonThreadFactory("blob-head"));
        try {
            scheduledExecutor.schedule(new Runnable() {
                @Override
                public void run() {
                    try {
                        outputStream.write(new byte[]{0x66, 0x67, 0x68, 0x69});
                    } catch (IOException ex) {
                        //pass
                    }
                }
            }, 800, TimeUnit.MILLISECONDS);
            PutHeadChunkRunnable runnable = new PutHeadChunkRunnable(
                digestBlob, 5, transportService, blobTransferTarget, discoveryNode, transferId
            );

            @SuppressWarnings("unchecked")
            TransportFuture<TransportResponse.Empty> result = mock(TransportFuture.class);

            when(transportService.submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            )).thenReturn(result);

            runnable.run();

            verify(blobTransferTarget).putHeadChunkTransferFinished(transferId);
            verify(transportService, times(2)).submitRequest(
                eq(discoveryNode),
                eq(BlobHeadRequestHandler.Actions.PUT_BLOB_HEAD_CHUNK),
                any(TransportRequest.class),
                any(TransportRequestOptions.class),
                eq(EmptyTransportResponseHandler.INSTANCE_SAME)
            );
        } finally {
            scheduledExecutor.awaitTermination(1, TimeUnit.SECONDS);
            scheduledExecutor.shutdownNow();
        }
    }
}
 
Example 18
Source File: TestDirectoryPathCreationWatcher.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Test
public void testAsyncDirFinderShuffledDirsCreatedPeriodically() throws Exception {
  String testDataDir = "target" + File.separatorChar + UUID.randomUUID().toString();
  Files.createDirectories(Paths.get(testDataDir));
  final Collection<Path> paths = Arrays.asList(
      Paths.get(testDataDir + File.separatorChar + "d1"),
      Paths.get(testDataDir + File.separatorChar + "d2"),
      Paths.get(testDataDir + File.separatorChar + "d3"),
      Paths.get(testDataDir + File.separatorChar + "d4"),
      Paths.get(testDataDir + File.separatorChar + "d5"),
      Paths.get(testDataDir + File.separatorChar + "d6"),
      Paths.get(testDataDir + File.separatorChar + "d7"),
      Paths.get(testDataDir + File.separatorChar + "d8"),

      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d11"),
      Paths.get(testDataDir + File.separatorChar + "d1" + File.separatorChar + "d12"),

      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d21" + File.separatorChar + "d211"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d22" + File.separatorChar + "d221"),
      Paths.get(testDataDir + File.separatorChar + "d2" + File.separatorChar + "d23" + File.separatorChar + "d231"),

      Paths.get(testDataDir + File.separatorChar + "d3" + File.separatorChar + "d31"),

      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41111" + File.separatorChar + "d411111"),
      Paths.get(testDataDir + File.separatorChar + "d4" + File.separatorChar + "d41" + File.separatorChar + "d411"
          + File.separatorChar + "d4111" + File.separatorChar + "d41112" + File.separatorChar + "d411111")

  );
  DirectoryPathCreationWatcher watcher = new DirectoryPathCreationWatcher(paths, 1);

  ScheduledExecutorService directoryCreationExecutor = new SafeScheduledExecutorService(1, "Directory Creator");
  DirectoryCreationWorker worker = new DirectoryCreationWorker(paths, true);
  directoryCreationExecutor.scheduleAtFixedRate(worker, 0, 100, TimeUnit.MILLISECONDS);

  Set<Path> foundPaths = new HashSet<Path>();
  while (!worker.getRemainingPathsToCreate().isEmpty()) {
    Set<Path> foundPathsTillNow = watcher.find();
    foundPaths.addAll(foundPathsTillNow);
    Thread.sleep(100);
  }

  directoryCreationExecutor.shutdownNow();
  directoryCreationExecutor.awaitTermination(1000, TimeUnit.MILLISECONDS);

  Thread.sleep(paths.size() * 100 + 1000);

  //Just to find the last set of detected directories.
  foundPaths.addAll(watcher.find());

  checkCorrectness(paths, foundPaths, (ScheduledExecutorService)Whitebox.getInternalState(watcher, "executor"));
}
 
Example 19
Source File: TaskContinuousTest.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
/**
 * Test that a streaming task will work correctly when an extractor is continuously producing records and encounters
 * an error in the writer.
 *
 * The task should exit in a failed state.
 *
 * No converters
 * Identity fork
 * One writer
 * @throws Exception
 */
@Test
public void testContinuousTaskError()
    throws Exception {

  for (Boolean taskExecutionSync: new Boolean[]{true, false}) {
    ArrayList<Object> recordCollector = new ArrayList<>(100);
    long perRecordExtractLatencyMillis = 1000; // 1 second per record

    ContinuousExtractor continuousExtractor = new ContinuousExtractor(perRecordExtractLatencyMillis);

    TaskContext mockTaskContext = getMockTaskContext(recordCollector, continuousExtractor, taskExecutionSync, 5);

    // Create a mock TaskStateTracker
    TaskStateTracker mockTaskStateTracker = mock(TaskStateTracker.class);

    // Create a TaskExecutor - a real TaskExecutor must be created so a Fork is run in a separate thread
    TaskExecutor taskExecutor = new TaskExecutor(new Properties());

    // Create the Task
    Task task = new Task(mockTaskContext, mockTaskStateTracker, taskExecutor, Optional.<CountDownLatch>absent());

    ScheduledExecutorService taskRunner = new ScheduledThreadPoolExecutor(1, ExecutorsUtils.newThreadFactory(Optional.of(log)));

    taskRunner.execute(task);

    // Let the task run for 10 seconds
    int sleepIterations = 10;
    int currentIteration = 0;

    while (currentIteration < sleepIterations) {
      Thread.sleep(1000);
      currentIteration++;
      Map<String, CheckpointableWatermark> externalWatermarkStorage = mockTaskContext.getWatermarkStorage()
          .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"));
      if (!externalWatermarkStorage.isEmpty()) {
        for (CheckpointableWatermark watermark : externalWatermarkStorage.values()) {
          log.info("Observed committed watermark: {}", watermark);
        }
        log.info("Task progress: {}", task.getProgress());
        // Ensure that watermarks seem reasonable at each step
        Assert.assertTrue(continuousExtractor.validateWatermarks(false, externalWatermarkStorage));
      }
    }

    boolean success = task.awaitShutdown(30000);
    Assert.assertTrue(success, "Task should shutdown in 30 seconds");
    log.info("Task done waiting to shutdown {}", success);

    // Shutdown on error, so don't check for the exact watermark
    Assert.assertTrue(continuousExtractor.validateWatermarks(false, mockTaskContext.getWatermarkStorage()
        .getCommittedWatermarks(CheckpointableWatermark.class, ImmutableList.of("default"))));

    task.commit();

    // Task should be in failed state from extractor error
    Assert.assertEquals(mockTaskContext.getTaskState().getWorkingState(), WorkUnitState.WorkingState.FAILED);
    // Shutdown the executor
    taskRunner.shutdown();
    taskRunner.awaitTermination(100, TimeUnit.MILLISECONDS);
  }
}
 
Example 20
Source File: StandardFlowService.java    From nifi with Apache License 2.0 4 votes vote down vote up
@Override
public void stop(final boolean force) {
    writeLock.lock();
    try {

        if (!isRunning()) {
            return;
        }

        running.set(false);

        if (clusterCoordinator != null) {
            final Thread shutdownClusterCoordinator = new Thread(clusterCoordinator::shutdown);

            shutdownClusterCoordinator.setDaemon(true);
            shutdownClusterCoordinator.setName("Shutdown Cluster Coordinator");
            shutdownClusterCoordinator.start();
        }

        if (!controller.isTerminated()) {
            controller.shutdown(force);
        }

        if (configuredForClustering && senderListener != null) {
            try {
                senderListener.stop();
            } catch (final IOException ioe) {
                logger.warn("Protocol sender/listener did not stop gracefully due to: " + ioe);
            }
        }

        final ScheduledExecutorService executorService = executor.get();
        if (executorService != null) {
            if (force) {
                executorService.shutdownNow();
            } else {
                executorService.shutdown();
            }

            boolean graceful;
            try {
                graceful = executorService.awaitTermination(gracefulShutdownSeconds, TimeUnit.SECONDS);
            } catch (final InterruptedException e) {
                graceful = false;
            }

            if (!graceful) {
                logger.warn("Scheduling service did not gracefully shutdown within configured " + gracefulShutdownSeconds + " second window");
            }
        }
    } finally {
        writeLock.unlock();
    }
}