io.airlift.units.Duration Java Examples

public BucketBalancer(
        NodeSupplier nodeSupplier,
        ShardManager shardManager,
        boolean enabled,
        Duration interval,
        boolean backupAvailable,
        boolean coordinator,
        String connectorId)
{
    this.nodeSupplier = requireNonNull(nodeSupplier, "nodeSupplier is null");
    this.shardManager = requireNonNull(shardManager, "shardManager is null");
    this.enabled = enabled;
    this.interval = requireNonNull(interval, "interval is null");
    this.backupAvailable = backupAvailable;
    this.coordinator = coordinator;
    this.executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("bucket-balancer-" + connectorId));
}
 

public static String formatDataRate(DataSize dataSize, Duration duration, boolean longForm)
{
    long rate = Math.round(dataSize.toBytes() / duration.getValue(SECONDS));
    if (Double.isNaN(rate) || Double.isInfinite(rate)) {
        rate = 0;
    }

    String rateString = formatDataSize(DataSize.ofBytes(rate), false);
    if (longForm) {
        if (!rateString.endsWith("B")) {
            rateString += "B";
        }
        rateString += "/s";
    }
    return rateString;
}
 

@Test
public void testExplicitPropertyMappings()
{
    Map<String, String> properties = new ImmutableMap.Builder<String, String>()
            .put("mysql.auto-reconnect", "false")
            .put("mysql.max-reconnects", "4")
            .put("mysql.connection-timeout", "4s")
            .put("mysql.jdbc.use-information-schema", "false")
            .build();

    MySqlConfig expected = new MySqlConfig()
            .setAutoReconnect(false)
            .setMaxReconnects(4)
            .setConnectionTimeout(new Duration(4, TimeUnit.SECONDS))
            .setDriverUseInformationSchema(false);

    assertFullMapping(properties, expected);
}
 

@Test
public void testExplicitPropertyMappings()
{
    Map<String, String> properties = new ImmutableMap.Builder<String, String>()
            .put("thrift.client.thread-count", "99")
            .put("thrift.client.connection-pool.enabled", "false")
            .put("thrift.client.connection-pool.max-size", "555")
            .put("thrift.client.connection-pool.idle-timeout", "7m")
            .put("thrift.client.ssl-context.refresh-time", "33m")
            .put("thrift.client.socks-proxy", "example.com:9876")
            .build();

    DriftNettyConnectionFactoryConfig expected = new DriftNettyConnectionFactoryConfig()
            .setThreadCount(99)
            .setConnectionPoolEnabled(false)
            .setConnectionPoolMaxSize(555)
            .setConnectionPoolIdleTimeout(new Duration(7, MINUTES))
            .setSslContextRefreshTime(new Duration(33, MINUTES))
            .setSocksProxy(HostAndPort.fromParts("example.com", 9876));

    assertFullMapping(properties, expected);
}
 

public PreviewTableCache(final QueryRunner.QueryRunnerFactory queryRunnerFactory,
                         final Duration previewCacheLifetime,
                         final ExecutorService executor,
                         final int previewLimit)
{
    this.queryRunnerFactory = checkNotNull(queryRunnerFactory, "queryRunnerFactory session was null!");

    ListeningExecutorService listeningExecutor = MoreExecutors.listeningDecorator(executor);

    BackgroundCacheLoader<PartitionedTableWithValue, List<List<Object>>> tableLoader =
            new BackgroundCacheLoader<PartitionedTableWithValue, List<List<Object>>>(listeningExecutor)
    {
                @Override
                public List<List<Object>> load(PartitionedTableWithValue key)
                        throws Exception
                {
                    return queryRows(buildQueryWithLimit(key, previewLimit));
                }
    };

    this.previewTableCache = CacheBuilder.newBuilder()
                                        .expireAfterWrite(Math.round(previewCacheLifetime.getValue()),
                                                          previewCacheLifetime.getUnit())
                                        .maximumSize(previewLimit)
                                        .build(tableLoader);
}
 

@Test(dependsOnMethods = "testRetrieveUpValue")
public void testCorrectNumberOfSplitsCreated()
        throws Exception
{
    PrometheusConnectorConfig config = new PrometheusConnectorConfig();
    config.setPrometheusURI(new URI("http://" + server.getAddress().getHost() + ":" + server.getAddress().getPort() + "/"));
    config.setMaxQueryRangeDuration(Duration.valueOf("21d"));
    config.setQueryChunkSizeDuration(Duration.valueOf("1d"));
    config.setCacheDuration(Duration.valueOf("30s"));
    PrometheusTable table = client.getTable("default", "up");
    PrometheusSplitManager splitManager = new PrometheusSplitManager(client, config);
    ConnectorSplitSource splits = splitManager.getSplits(
            null,
            null,
            (ConnectorTableHandle) new PrometheusTableHandle("default", table.getName()),
            null);
    int numSplits = splits.getNextBatch(NOT_PARTITIONED, NUMBER_MORE_THAN_EXPECTED_NUMBER_SPLITS).getNow(null).getSplits().size();
    assertEquals(numSplits, config.getMaxQueryRangeDuration().getValue(TimeUnit.SECONDS) / config.getQueryChunkSizeDuration().getValue(TimeUnit.SECONDS),
            0.001);
}
 

@Test
public void testGetServerInfo()
        throws Exception
{
    ServerInfo expected = new ServerInfo(UNKNOWN, "test", true, false, Optional.of(Duration.valueOf("2m")));

    server.enqueue(new MockResponse()
            .addHeader(CONTENT_TYPE, "application/json")
            .setBody(SERVER_INFO_CODEC.toJson(expected)));

    QueryExecutor executor = new QueryExecutor(new OkHttpClient());

    ServerInfo actual = executor.getServerInfo(server.url("/v1/info").uri());
    assertEquals(actual.getEnvironment(), "test");
    assertEquals(actual.getUptime(), Optional.of(Duration.valueOf("2m")));

    assertEquals(server.getRequestCount(), 1);
    assertEquals(server.takeRequest().getPath(), "/v1/info");
}
 

public SemiTransactionalHiveMetastore(
        HdfsEnvironment hdfsEnvironment,
        HiveMetastoreClosure delegate,
        Executor renameExecutor,
        Executor dropExecutor,
        boolean skipDeletionForAlter,
        boolean skipTargetCleanupOnRollback,
        Optional<Duration> hiveTransactionHeartbeatInterval,
        ScheduledExecutorService heartbeatService)
{
    this.hdfsEnvironment = requireNonNull(hdfsEnvironment, "hdfsEnvironment is null");
    this.delegate = requireNonNull(delegate, "delegate is null");
    this.renameExecutor = requireNonNull(renameExecutor, "renameExecutor is null");
    this.dropExecutor = requireNonNull(dropExecutor, "dropExecutor is null");
    this.skipDeletionForAlter = skipDeletionForAlter;
    this.skipTargetCleanupOnRollback = skipTargetCleanupOnRollback;
    this.heartbeatExecutor = heartbeatService;
    this.configuredTransactionHeartbeatInterval = requireNonNull(hiveTransactionHeartbeatInterval, "hiveTransactionHeartbeatInterval is null");
}
 

private static DistributedQueryRunner createQueryRunner()
        throws Exception
{
    Session session = testSessionBuilder().build();
    assertEquals(session.getSystemProperties(), emptyMap());

    Duration sessionValue = session.getSystemProperty(EXAMPLE_PROPERTY, Duration.class);
    assertEquals(sessionValue, EXAMPLE_VALUE_DEFAULT);
    assertNotEquals(EXAMPLE_VALUE_DEFAULT, EXAMPLE_VALUE_CONFIGURED);

    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
    queryRunner.installPlugin(new TestingSessionPropertyConfigurationManagerPlugin());
    return queryRunner;
}
 

@Config("prometheus.cache.ttl")
@ConfigDescription("How long values from this config file are cached")
public PrometheusConnectorConfig setCacheDuration(Duration cacheConfigDuration)
{
    this.cacheDuration = cacheConfigDuration;
    return this;
}
 

@ConfigDescription("Timeout for queries to the control cluster")
@Config("control.timeout")
public VerifierConfig setControlTimeout(Duration controlTimeout)
{
    this.controlTimeout = controlTimeout;
    return this;
}
 

private void assertSessionPropertyValue(String user, Duration expectedValue)
{
    Session session = testSessionBuilder()
            .setIdentity(Identity.ofUser(user))
            .build();

    MaterializedResult result = queryRunner.execute(session, "SHOW SESSION");
    String actualValueString = (String) result.getMaterializedRows().stream()
            .filter(row -> (row.getField(0).equals(EXAMPLE_PROPERTY)))
            .collect(onlyElement())
            .getField(1);

    assertEquals(Duration.valueOf(actualValueString), expectedValue);
}
 

@Config("transaction.idle-check-interval")
@ConfigDescription("Time interval between idle transactions checks")
public TransactionManagerConfig setIdleCheckInterval(Duration idleCheckInterval)
{
    this.idleCheckInterval = idleCheckInterval;
    return this;
}
 

@Config("elasticsearch.request-timeout")
@ConfigDescription("Elasticsearch request timeout")
public ElasticsearchConfig setRequestTimeout(Duration requestTimeout)
{
    this.requestTimeout = requestTimeout;
    return this;
}
 

@JsonCreator
public ServerInfo(
        @JsonProperty("nodeVersion") NodeVersion nodeVersion,
        @JsonProperty("environment") String environment,
        @JsonProperty("coordinator") boolean coordinator,
        @JsonProperty("starting") boolean starting,
        @JsonProperty("uptime") Optional<Duration> uptime)
{
    this.nodeVersion = requireNonNull(nodeVersion, "nodeVersion is null");
    this.environment = requireNonNull(environment, "environment is null");
    this.coordinator = coordinator;
    this.starting = starting;
    this.uptime = requireNonNull(uptime, "uptime is null");
}
 

@Test
public void testFailureInterval()
{
    TestingTicker ticker = new TestingTicker();
    ticker.increment(1, NANOSECONDS);

    Backoff backoff = new Backoff(1, new Duration(15, SECONDS), ticker, ImmutableList.of(new Duration(10, MILLISECONDS)));
    ticker.increment(10, MICROSECONDS);

    // verify initial state
    assertEquals(backoff.getFailureCount(), 0);
    assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0);

    // first failure, should never fail
    assertFalse(backoff.failure());
    assertEquals(backoff.getFailureCount(), 1);
    assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 0);

    ticker.increment(14, SECONDS);

    // second failure within the limit, should not fail
    assertFalse(backoff.failure());
    assertEquals(backoff.getFailureCount(), 2);
    assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 14);

    ticker.increment(1, SECONDS);

    // final failure after the limit causes failure
    assertTrue(backoff.failure());
    assertEquals(backoff.getFailureCount(), 3);
    assertEquals(backoff.getFailureDuration().roundTo(SECONDS), 15);
}
 

public static String formatFinalTime(Duration duration)
{
    long totalMillis = duration.toMillis();

    if (totalMillis >= MINUTES.toMillis(1)) {
        return formatTime(duration);
    }

    return format("%.2f", (totalMillis / 1000.0));
}
 

@Override
public void run()
{
    try {
        stats.addQueuedTime(Duration.nanosSince(queuedTime));
        long start = System.nanoTime();

        backupStore.get().backupShard(uuid, source);
        stats.addCopyShardDataRate(DataSize.ofBytes(source.length()), Duration.nanosSince(start));

        File restored = new File(storageService.getStagingFile(uuid) + ".validate");
        backupStore.get().restoreShard(uuid, restored);

        if (!filesEqual(source, restored)) {
            stats.incrementBackupCorruption();

            File quarantineBase = storageService.getQuarantineFile(uuid);
            File quarantineOriginal = new File(quarantineBase.getPath() + ".original");
            File quarantineRestored = new File(quarantineBase.getPath() + ".restored");

            log.error("Backup is corrupt after write. Quarantining local file: %s", quarantineBase);
            if (!this.source.renameTo(quarantineOriginal) || !restored.renameTo(quarantineRestored)) {
                log.warn("Quarantine of corrupt backup shard failed: %s", uuid);
            }

            throw new PrestoException(RAPTOR_BACKUP_CORRUPTION, "Backup is corrupt after write: " + uuid);
        }

        if (!restored.delete()) {
            log.warn("Failed to delete staging file: %s", restored);
        }

        stats.incrementBackupSuccess();
    }
    catch (Throwable t) {
        stats.incrementBackupFailure();
        throw t;
    }
}
 

@Test
public void testRampup()
{
    SplitConcurrencyController controller = new SplitConcurrencyController(1, new Duration(1, SECONDS));
    for (int i = 0; i < 10; i++) {
        controller.update(SECONDS.toNanos(2), 0, i + 1);
        assertEquals(controller.getTargetConcurrency(), i + 2);
    }
}
 

private synchronized void nextAttempt(boolean noConnectDelay)
{
    try {
        // request was already canceled
        if (isCancelled()) {
            return;
        }

        Optional<A> address = addressSelector.selectAddress(addressSelectionContext, attemptedAddresses);
        if (!address.isPresent()) {
            fail("No hosts available");
            return;
        }

        if (invocationAttempts > 0) {
            stat.recordRetry();
        }

        if (noConnectDelay) {
            invoke(address.get());
            return;
        }

        int connectionFailuresCount = failedConnectionAttempts.count(address.get());
        if (connectionFailuresCount == 0) {
            invoke(address.get());
            return;
        }

        Duration connectDelay = retryPolicy.getBackoffDelay(connectionFailuresCount);
        log.debug("Failed connection to %s with attempt %s, will retry in %s", address.get(), connectionFailuresCount, connectDelay);
        schedule(connectDelay, () -> invoke(address.get()));
    }
    catch (Throwable t) {
        // this should never happen, but ensure that invocation always finishes
        unexpectedError(t);
    }
}
 

private static Backoff createCleanupBackoff()
{
    return new Backoff(10, new Duration(10, TimeUnit.MINUTES), Ticker.systemTicker(), ImmutableList.<Duration>builder()
            .add(new Duration(0, MILLISECONDS))
            .add(new Duration(100, MILLISECONDS))
            .add(new Duration(500, MILLISECONDS))
            .add(new Duration(1, SECONDS))
            .add(new Duration(10, SECONDS))
            .build());
}
 

private void splitFinished(PrioritizedSplitRunner split)
{
    completedSplitsPerLevel.incrementAndGet(split.getPriority().getLevel());
    synchronized (this) {
        allSplits.remove(split);

        long wallNanos = System.nanoTime() - split.getCreatedNanos();
        splitWallTime.add(Duration.succinctNanos(wallNanos));

        if (intermediateSplits.remove(split)) {
            intermediateSplitWallTime.add(wallNanos);
            intermediateSplitScheduledTime.add(split.getScheduledNanos());
            intermediateSplitWaitTime.add(split.getWaitNanos());
            intermediateSplitCpuTime.add(split.getCpuTimeNanos());
        }
        else {
            leafSplitWallTime.add(wallNanos);
            leafSplitScheduledTime.add(split.getScheduledNanos());
            leafSplitWaitTime.add(split.getWaitNanos());
            leafSplitCpuTime.add(split.getCpuTimeNanos());
        }

        TaskHandle taskHandle = split.getTaskHandle();
        taskHandle.splitComplete(split);

        scheduleTaskIfNecessary(taskHandle);

        addNewEntrants();
    }
    // call destroy outside of synchronized block as it is expensive and doesn't need a lock on the task executor
    split.destroy();
}
 

@Override
public BasicQueryInfo getBasicQueryInfo()
{
    return new BasicQueryInfo(
            new QueryId("test"),
            session.toSessionRepresentation(),
            Optional.empty(),
            state,
            new MemoryPoolId("test"),
            !state.isDone(),
            URI.create("http://test"),
            "SELECT 1",
            Optional.empty(),
            Optional.empty(),
            new BasicQueryStats(
                    new DateTime(1),
                    new DateTime(2),
                    new Duration(3, NANOSECONDS),
                    new Duration(4, NANOSECONDS),
                    new Duration(5, NANOSECONDS),
                    6,
                    7,
                    8,
                    9,
                    DataSize.ofBytes(14),
                    15,
                    16.0,
                    memoryUsage,
                    memoryUsage,
                    DataSize.ofBytes(19),
                    DataSize.ofBytes(20),
                    cpuUsage,
                    new Duration(22, NANOSECONDS),
                    false,
                    ImmutableSet.of(),
                    OptionalDouble.empty()),
            null,
            null);
}
 

private synchronized void scheduleTaskIfNecessary(TaskHandle taskHandle)
{
    // if task has less than the minimum guaranteed splits running,
    // immediately schedule a new split for this task.  This assures
    // that a task gets its fair amount of consideration (you have to
    // have splits to be considered for running on a thread).
    if (taskHandle.getRunningLeafSplits() < min(guaranteedNumberOfDriversPerTask, taskHandle.getMaxDriversPerTask().orElse(Integer.MAX_VALUE))) {
        PrioritizedSplitRunner split = taskHandle.pollNextSplit();
        if (split != null) {
            startSplit(split);
            splitQueuedTime.add(Duration.nanosSince(split.getCreatedNanos()));
        }
    }
}
 

@JsonCreator
public BlackHoleOutputTableHandle(
        @JsonProperty("table") BlackHoleTableHandle table,
        @JsonProperty("pageProcessingDelay") Duration pageProcessingDelay)
{
    this.table = requireNonNull(table, "table is null");
    this.pageProcessingDelay = requireNonNull(pageProcessingDelay, "pageProcessingDelay is null");
}
 

@Test
public void testUppercaseNameEscaped()
{
    /*
     * If an identifier is escaped with double quotes it is stored verbatim
     *
     * http://docs.datastax.com/en/cql/3.1/cql/cql_reference/ucase-lcase_r.html
     */
    session.execute("CREATE KEYSPACE \"KEYSPACE_2\" WITH REPLICATION = {'class':'SimpleStrategy', 'replication_factor': 1}");
    assertContainsEventually(() -> execute("SHOW SCHEMAS FROM cassandra"), resultBuilder(getSession(), createUnboundedVarcharType())
            .row("keyspace_2")
            .build(), new Duration(1, MINUTES));

    session.execute("CREATE TABLE \"KEYSPACE_2\".\"TABLE_2\" (\"COLUMN_2\" bigint PRIMARY KEY)");
    assertContainsEventually(() -> execute("SHOW TABLES FROM cassandra.keyspace_2"), resultBuilder(getSession(), createUnboundedVarcharType())
            .row("table_2")
            .build(), new Duration(1, MINUTES));
    assertContains(execute("SHOW COLUMNS FROM cassandra.keyspace_2.table_2"), resultBuilder(getSession(), createUnboundedVarcharType(), createUnboundedVarcharType(), createUnboundedVarcharType(), createUnboundedVarcharType())
            .row("column_2", "bigint", "", "")
            .build());

    execute("INSERT INTO \"KEYSPACE_2\".\"TABLE_2\" (\"COLUMN_2\") VALUES (1)");

    assertEquals(execute("SELECT column_2 FROM cassandra.keyspace_2.table_2").getRowCount(), 1);
    assertUpdate("DROP TABLE cassandra.keyspace_2.table_2");

    // when an identifier is unquoted the lowercase and uppercase spelling may be used interchangeable
    session.execute("DROP KEYSPACE \"KEYSPACE_2\"");
}
 

public Duration getElapsedTime()
{
    if (endNanos.get() != null) {
        return succinctNanos(endNanos.get() - createNanos);
    }
    return nanosSince(createNanos, tickerNanos());
}
 

public ShardCleaner(
        DaoSupplier<ShardDao> shardDaoSupplier,
        String currentNode,
        boolean coordinator,
        Ticker ticker,
        StorageService storageService,
        Optional<BackupStore> backupStore,
        Duration maxTransactionAge,
        Duration transactionCleanerInterval,
        Duration localCleanerInterval,
        Duration localCleanTime,
        Duration backupCleanerInterval,
        Duration backupCleanTime,
        int backupDeletionThreads,
        Duration maxCompletedTransactionAge)
{
    this.dao = shardDaoSupplier.onDemand();
    this.currentNode = requireNonNull(currentNode, "currentNode is null");
    this.coordinator = coordinator;
    this.ticker = requireNonNull(ticker, "ticker is null");
    this.storageService = requireNonNull(storageService, "storageService is null");
    this.backupStore = requireNonNull(backupStore, "backupStore is null");
    this.maxTransactionAge = requireNonNull(maxTransactionAge, "maxTransactionAge");
    this.transactionCleanerInterval = requireNonNull(transactionCleanerInterval, "transactionCleanerInterval is null");
    this.localCleanerInterval = requireNonNull(localCleanerInterval, "localCleanerInterval is null");
    this.localCleanTime = requireNonNull(localCleanTime, "localCleanTime is null");
    this.backupCleanerInterval = requireNonNull(backupCleanerInterval, "backupCleanerInterval is null");
    this.backupCleanTime = requireNonNull(backupCleanTime, "backupCleanTime is null");
    this.scheduler = newScheduledThreadPool(2, daemonThreadsNamed("shard-cleaner-%s"));
    this.backupExecutor = newFixedThreadPool(backupDeletionThreads, daemonThreadsNamed("shard-cleaner-backup-%s"));
    this.maxCompletedTransactionAge = requireNonNull(maxCompletedTransactionAge, "maxCompletedTransactionAge is null");
}
 

@Test
public void testDefault()
{
    assertRecordedDefaults(recordDefaults(LdapConfig.class)
            .setLdapUrl(null)
            .setAllowInsecure(false)
            .setTrustCertificate(null)
            .setUserBindSearchPattern(null)
            .setUserBaseDistinguishedName(null)
            .setGroupAuthorizationSearchPattern(null)
            .setBindDistingushedName(null)
            .setBindPassword(null)
            .setIgnoreReferrals(false)
            .setLdapCacheTtl(new Duration(1, TimeUnit.HOURS)));
}
 

public static PropertyMetadata<Duration> durationProperty(String name, String description, Duration defaultValue, boolean hidden)
{
    return new PropertyMetadata<>(
            name,
            description,
            VARCHAR,
            Duration.class,
            defaultValue,
            hidden,
            value -> Duration.valueOf((String) value),
            Duration::toString);
}