io.airlift.units.DataSize Java Examples

private OrcWriterOptions(
        DataSize stripeMinSize,
        DataSize stripeMaxSize,
        int stripeMaxRowCount,
        int rowGroupMaxRowCount,
        DataSize dictionaryMaxMemory,
        DataSize maxStringStatisticsLimit,
        DataSize maxCompressionBufferSize)
{
    requireNonNull(stripeMinSize, "stripeMinSize is null");
    requireNonNull(stripeMaxSize, "stripeMaxSize is null");
    checkArgument(stripeMaxRowCount >= 1, "stripeMaxRowCount must be at least 1");
    checkArgument(rowGroupMaxRowCount >= 1, "rowGroupMaxRowCount must be at least 1");
    requireNonNull(dictionaryMaxMemory, "dictionaryMaxMemory is null");
    requireNonNull(maxStringStatisticsLimit, "maxStringStatisticsLimit is null");
    requireNonNull(maxCompressionBufferSize, "maxCompressionBufferSize is null");

    this.stripeMinSize = stripeMinSize;
    this.stripeMaxSize = stripeMaxSize;
    this.stripeMaxRowCount = stripeMaxRowCount;
    this.rowGroupMaxRowCount = rowGroupMaxRowCount;
    this.dictionaryMaxMemory = dictionaryMaxMemory;
    this.maxStringStatisticsLimit = maxStringStatisticsLimit;
    this.maxCompressionBufferSize = maxCompressionBufferSize;
}
 

public TypedSet(Type elementType, Optional<MethodHandle> elementIsDistinctFrom, BlockBuilder blockBuilder, int expectedSize, String functionName, Optional<DataSize> maxBlockMemory)
{
    checkArgument(expectedSize >= 0, "expectedSize must not be negative");
    this.elementType = requireNonNull(elementType, "elementType must not be null");
    this.elementIsDistinctFrom = requireNonNull(elementIsDistinctFrom, "elementIsDistinctFrom is null");
    elementIsDistinctFrom.ifPresent(methodHandle -> checkArgument(methodHandle.type().equals(MethodType.methodType(boolean.class, Block.class, int.class, Block.class, int.class))));
    this.elementBlock = requireNonNull(blockBuilder, "blockBuilder must not be null");
    this.functionName = functionName;
    this.maxBlockMemoryInBytes = maxBlockMemory.map(value -> value.toBytes()).orElse(Long.MAX_VALUE);

    initialElementBlockOffset = elementBlock.getPositionCount();
    initialElementBlockSizeInBytes = elementBlock.getSizeInBytes();

    this.size = 0;
    this.hashCapacity = arraySize(expectedSize, FILL_RATIO);
    this.maxFill = calculateMaxFill(hashCapacity);
    this.hashMask = hashCapacity - 1;

    blockPositionByHash = new IntArrayList(hashCapacity);
    blockPositionByHash.size(hashCapacity);
    for (int i = 0; i < hashCapacity; i++) {
        blockPositionByHash.set(i, EMPTY_SLOT);
    }

    this.containsNullElement = false;
}
 

public PartitionedOutputFactory(
        PartitionFunction partitionFunction,
        List<Integer> partitionChannels,
        List<Optional<NullableValue>> partitionConstants,
        boolean replicatesAnyRow,
        OptionalInt nullChannel,
        OutputBuffer outputBuffer,
        DataSize maxMemory)
{
    this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null");
    this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null");
    this.partitionConstants = requireNonNull(partitionConstants, "partitionConstants is null");
    this.replicatesAnyRow = replicatesAnyRow;
    this.nullChannel = requireNonNull(nullChannel, "nullChannel is null");
    this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null");
    this.maxMemory = requireNonNull(maxMemory, "maxMemory is null");
}
 

private HiveSplitSource(
        ConnectorSession session,
        String databaseName,
        String tableName,
        PerBucket queues,
        int maxInitialSplits,
        DataSize maxOutstandingSplitsSize,
        HiveSplitLoader splitLoader,
        AtomicReference<State> stateReference,
        CounterStat highMemorySplitSourceCounter)
{
    requireNonNull(session, "session is null");
    this.queryId = session.getQueryId();
    this.databaseName = requireNonNull(databaseName, "databaseName is null");
    this.tableName = requireNonNull(tableName, "tableName is null");
    this.queues = requireNonNull(queues, "queues is null");
    this.maxOutstandingSplitsBytes = requireNonNull(maxOutstandingSplitsSize, "maxOutstandingSplitsSize is null").toBytes();
    this.splitLoader = requireNonNull(splitLoader, "splitLoader is null");
    this.stateReference = requireNonNull(stateReference, "stateReference is null");
    this.highMemorySplitSourceCounter = requireNonNull(highMemorySplitSourceCounter, "highMemorySplitSourceCounter is null");

    this.maxSplitSize = getMaxSplitSize(session);
    this.maxInitialSplitSize = getMaxInitialSplitSize(session);
    this.remainingInitialSplits = new AtomicInteger(maxInitialSplits);
}
 

private void configureMemoryPools(NodeMemoryConfig config, long availableMemory)
{
    validateHeapHeadroom(config, availableMemory);
    maxMemory = DataSize.ofBytes(availableMemory - config.getHeapHeadroom().toBytes());
    checkArgument(
            config.getMaxQueryMemoryPerNode().toBytes() <= config.getMaxQueryTotalMemoryPerNode().toBytes(),
            "Max query memory per node (%s) cannot be greater than the max query total memory per node (%s).",
            QUERY_MAX_MEMORY_PER_NODE_CONFIG,
            QUERY_MAX_TOTAL_MEMORY_PER_NODE_CONFIG);
    ImmutableMap.Builder<MemoryPoolId, MemoryPool> builder = ImmutableMap.builder();
    long generalPoolSize = maxMemory.toBytes();
    if (!config.isReservedPoolDisabled()) {
        builder.put(RESERVED_POOL, new MemoryPool(RESERVED_POOL, config.getMaxQueryTotalMemoryPerNode()));
        generalPoolSize -= config.getMaxQueryTotalMemoryPerNode().toBytes();
    }
    verify(generalPoolSize > 0, "general memory pool size is 0");
    builder.put(GENERAL_POOL, new MemoryPool(GENERAL_POOL, DataSize.ofBytes(generalPoolSize)));
    this.pools = builder.build();
}
 

public static OperatorFactory createOperatorFactory(
        int operatorId,
        PlanNodeId planNodeId,
        Supplier<PageProcessor> processor,
        List<Type> types,
        DataSize minOutputPageSize,
        int minOutputPageRowCount)
{
    return createAdapterOperatorFactory(new Factory(
            operatorId,
            planNodeId,
            processor,
            types,
            minOutputPageSize,
            minOutputPageRowCount));
}
 

@Test
public void testMergeGap()
{
    List<DiskRange> consistent10ByteGap = ImmutableList.of(new DiskRange(100, 90), new DiskRange(200, 90), new DiskRange(300, 90));
    assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap);
    assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), consistent10ByteGap);
    assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290)));
    assertEquals(mergeAdjacentDiskRanges(consistent10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 290)));

    List<DiskRange> middle10ByteGap = ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 90), new DiskRange(300, 80), new DiskRange(400, 90));
    assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(0, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap);
    assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(9, BYTE), new DataSize(1, GIGABYTE)), middle10ByteGap);
    assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(10, BYTE), new DataSize(1, GIGABYTE)),
            ImmutableList.of(new DiskRange(100, 80), new DiskRange(200, 180), new DiskRange(400, 90)));
    assertEquals(mergeAdjacentDiskRanges(middle10ByteGap, new DataSize(100, BYTE), new DataSize(1, GIGABYTE)), ImmutableList.of(new DiskRange(100, 390)));
}
 

public IndexLookupSourceFactory(
        Set<Integer> lookupSourceInputChannels,
        List<Integer> keyOutputChannels,
        OptionalInt keyOutputHashChannel,
        List<Type> outputTypes,
        IndexBuildDriverFactoryProvider indexBuildDriverFactoryProvider,
        DataSize maxIndexMemorySize,
        IndexJoinLookupStats stats,
        boolean shareIndexLoading,
        PagesIndex.Factory pagesIndexFactory,
        JoinCompiler joinCompiler)
{
    this.outputTypes = ImmutableList.copyOf(requireNonNull(outputTypes, "outputTypes is null"));

    if (shareIndexLoading) {
        IndexLoader shared = new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler);
        this.indexLoaderSupplier = () -> shared;
    }
    else {
        this.indexLoaderSupplier = () -> new IndexLoader(lookupSourceInputChannels, keyOutputChannels, keyOutputHashChannel, outputTypes, indexBuildDriverFactoryProvider, 10_000, maxIndexMemorySize, stats, pagesIndexFactory, joinCompiler);
    }
}
 

@Test
public void testExplicitPropertyMappings()
{
    Map<String, String> properties = new ImmutableMap.Builder<String, String>()
            .put("exchange.max-buffer-size", "1GB")
            .put("exchange.concurrent-request-multiplier", "13")
            .put("exchange.min-error-duration", "13s")
            .put("exchange.max-error-duration", "33s")
            .put("exchange.max-response-size", "1MB")
            .put("exchange.client-threads", "2")
            .put("exchange.page-buffer-client.max-callback-threads", "16")
            .put("exchange.acknowledge-pages", "false")
            .build();

    ExchangeClientConfig expected = new ExchangeClientConfig()
            .setMaxBufferSize(DataSize.of(1, Unit.GIGABYTE))
            .setConcurrentRequestMultiplier(13)
            .setMinErrorDuration(new Duration(33, TimeUnit.SECONDS))
            .setMaxErrorDuration(new Duration(33, TimeUnit.SECONDS))
            .setMaxResponseSize(DataSize.of(1, Unit.MEGABYTE))
            .setClientThreads(2)
            .setPageBufferClientMaxCallbackThreads(16)
            .setAcknowledgePages(false);

    assertFullMapping(properties, expected);
}
 

private synchronized void updatePools(Map<MemoryPoolId, Integer> queryCounts)
{
    // Update view of cluster memory and pools
    List<MemoryInfo> nodeMemoryInfos = nodes.values().stream()
            .map(RemoteNodeMemory::getInfo)
            .filter(Optional::isPresent)
            .map(Optional::get)
            .collect(toImmutableList());

    long totalProcessors = nodeMemoryInfos.stream()
            .mapToLong(MemoryInfo::getAvailableProcessors)
            .sum();
    totalAvailableProcessors.set(totalProcessors);

    long totalClusterMemory = nodeMemoryInfos.stream()
            .map(MemoryInfo::getTotalNodeMemory)
            .mapToLong(DataSize::toBytes)
            .sum();
    clusterMemoryBytes.set(totalClusterMemory);

    for (ClusterMemoryPool pool : pools.values()) {
        pool.update(nodeMemoryInfos, queryCounts.getOrDefault(pool.getId(), 0));
        if (changeListeners.containsKey(pool.getId())) {
            MemoryPoolInfo info = pool.getInfo();
            for (Consumer<MemoryPoolInfo> listener : changeListeners.get(pool.getId())) {
                listenerExecutor.execute(() -> listener.accept(info));
            }
        }
    }
}
 

private void doIntegration(TestingOrcDataSource orcDataSource, DataSize maxMergeDistance, DataSize tinyStripeThreshold)
        throws IOException
{
    OrcReaderOptions options = new OrcReaderOptions()
            .withMaxMergeDistance(maxMergeDistance)
            .withTinyStripeThreshold(tinyStripeThreshold)
            .withMaxReadBlockSize(DataSize.of(1, Unit.MEGABYTE));
    OrcReader orcReader = new OrcReader(orcDataSource, options);
    // 1 for reading file footer
    assertEquals(orcDataSource.getReadCount(), 1);
    List<StripeInformation> stripes = orcReader.getFooter().getStripes();
    // Sanity check number of stripes. This can be three or higher because of orc writer low memory mode.
    assertGreaterThanOrEqual(stripes.size(), 3);
    //verify wrapped by CachingOrcReader
    assertInstanceOf(wrapWithCacheIfTinyStripes(orcDataSource, stripes, maxMergeDistance, tinyStripeThreshold), CachingOrcDataSource.class);

    OrcRecordReader orcRecordReader = orcReader.createRecordReader(
            orcReader.getRootColumn().getNestedColumns(),
            ImmutableList.of(VARCHAR),
            (numberOfRows, statisticsByColumnIndex) -> true,
            HIVE_STORAGE_TIME_ZONE,
            newSimpleAggregatedMemoryContext(),
            INITIAL_BATCH_SIZE,
            RuntimeException::new);
    int positionCount = 0;
    while (true) {
        Page page = orcRecordReader.nextPage();
        if (page == null) {
            break;
        }
        page = page.getLoadedPage();
        Block block = page.getBlock(0);
        positionCount += block.getPositionCount();
    }
    assertEquals(positionCount, POSITION_COUNT);
}
 

@Test
public void testMergeSingle()
{
    List<DiskRange> diskRanges = mergeAdjacentDiskRanges(
            ImmutableList.of(new DiskRange(100, 100)),
            new DataSize(0, BYTE),
            new DataSize(0, BYTE));
    assertEquals(diskRanges, ImmutableList.of(new DiskRange(100, 100)));
}
 

@Test(timeOut = 10_000)
public void testFairQueuing()
{
    InternalResourceGroup root = new InternalResourceGroup("root", (group, export) -> {}, directExecutor());
    root.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    root.setMaxQueuedQueries(4);
    root.setHardConcurrencyLimit(1);
    InternalResourceGroup group1 = root.getOrCreateSubGroup("1");
    group1.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    group1.setMaxQueuedQueries(4);
    group1.setHardConcurrencyLimit(2);
    InternalResourceGroup group2 = root.getOrCreateSubGroup("2");
    group2.setSoftMemoryLimitBytes(DataSize.of(1, MEGABYTE).toBytes());
    group2.setMaxQueuedQueries(4);
    group2.setHardConcurrencyLimit(2);
    MockManagedQueryExecution query1a = new MockManagedQueryExecutionBuilder().build();
    group1.run(query1a);
    assertEquals(query1a.getState(), RUNNING);
    MockManagedQueryExecution query1b = new MockManagedQueryExecutionBuilder().build();
    group1.run(query1b);
    assertEquals(query1b.getState(), QUEUED);
    MockManagedQueryExecution query1c = new MockManagedQueryExecutionBuilder().build();
    group1.run(query1c);
    assertEquals(query1c.getState(), QUEUED);
    MockManagedQueryExecution query2a = new MockManagedQueryExecutionBuilder().build();
    group2.run(query2a);
    assertEquals(query2a.getState(), QUEUED);

    query1a.complete();
    // 1b and not 2a should have started, as it became queued first and group1 was eligible to run more
    assertEquals(query1b.getState(), RUNNING);
    assertEquals(query1c.getState(), QUEUED);
    assertEquals(query2a.getState(), QUEUED);

    // 2a and not 1c should have started, as all eligible sub groups get fair sharing
    query1b.complete();
    assertEquals(query2a.getState(), RUNNING);
    assertEquals(query1c.getState(), QUEUED);
}
 

public PagePartitioner(
        PartitionFunction partitionFunction,
        List<Integer> partitionChannels,
        List<Optional<NullableValue>> partitionConstants,
        boolean replicatesAnyRow,
        OptionalInt nullChannel,
        OutputBuffer outputBuffer,
        PagesSerdeFactory serdeFactory,
        List<Type> sourceTypes,
        DataSize maxMemory,
        OperatorContext operatorContext)
{
    this.partitionFunction = requireNonNull(partitionFunction, "partitionFunction is null");
    this.partitionChannels = requireNonNull(partitionChannels, "partitionChannels is null");
    this.partitionConstants = requireNonNull(partitionConstants, "partitionConstants is null").stream()
            .map(constant -> constant.map(NullableValue::asBlock))
            .collect(toImmutableList());
    this.replicatesAnyRow = replicatesAnyRow;
    this.nullChannel = requireNonNull(nullChannel, "nullChannel is null");
    this.outputBuffer = requireNonNull(outputBuffer, "outputBuffer is null");
    this.sourceTypes = requireNonNull(sourceTypes, "sourceTypes is null");
    this.serde = requireNonNull(serdeFactory, "serdeFactory is null").createPagesSerde();
    this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");

    int partitionCount = partitionFunction.getPartitionCount();
    int pageSize = toIntExact(min(DEFAULT_MAX_PAGE_SIZE_IN_BYTES, maxMemory.toBytes() / partitionCount));
    pageSize = max(1, pageSize);

    this.pageBuilders = new PageBuilder[partitionCount];
    for (int i = 0; i < partitionCount; i++) {
        pageBuilders[i] = PageBuilder.withMaxPageSize(pageSize, sourceTypes);
    }
}
 

@Test
public void testCountAlreadyRevokedMemoryWithinAPool()
        throws Exception
{
    // Given
    SqlTask sqlTask1 = newSqlTask();
    MemoryPool anotherMemoryPool = new MemoryPool(new MemoryPoolId("test"), DataSize.ofBytes(10));
    sqlTask1.getQueryContext().setMemoryPool(anotherMemoryPool);
    OperatorContext operatorContext1 = createContexts(sqlTask1);

    SqlTask sqlTask2 = newSqlTask();
    OperatorContext operatorContext2 = createContexts(sqlTask2);

    List<SqlTask> tasks = ImmutableList.of(sqlTask1, sqlTask2);
    MemoryRevokingScheduler scheduler = new MemoryRevokingScheduler(asList(memoryPool, anotherMemoryPool), () -> tasks, executor, 1.0, 1.0);
    allOperatorContexts = ImmutableSet.of(operatorContext1, operatorContext2);

    /*
     * sqlTask1 fills its pool
     */
    operatorContext1.localRevocableMemoryContext().setBytes(12);
    requestMemoryRevoking(scheduler);
    assertMemoryRevokingRequestedFor(operatorContext1);

    /*
     * When sqlTask2 fills its pool
     */
    operatorContext2.localRevocableMemoryContext().setBytes(12);
    requestMemoryRevoking(scheduler);

    /*
     * Then sqlTask2 should be asked to revoke its memory too
     */
    assertMemoryRevokingRequestedFor(operatorContext1, operatorContext2);
}
 

public ShardCompactionManager(
        IDBI dbi,
        String currentNodeIdentifier,
        ShardManager shardManager,
        ShardOrganizer organizer,
        TemporalFunction temporalFunction,
        Duration compactionDiscoveryInterval,
        DataSize maxShardSize,
        long maxShardRows,
        boolean compactionEnabled)
{
    this.dbi = requireNonNull(dbi, "dbi is null");
    this.metadataDao = onDemandDao(dbi, MetadataDao.class);

    this.currentNodeIdentifier = requireNonNull(currentNodeIdentifier, "currentNodeIdentifier is null");
    this.shardManager = requireNonNull(shardManager, "shardManager is null");
    this.organizer = requireNonNull(organizer, "organizer is null");
    this.compactionDiscoveryInterval = requireNonNull(compactionDiscoveryInterval, "compactionDiscoveryInterval is null");

    checkArgument(maxShardSize.toBytes() > 0, "maxShardSize must be > 0");
    this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null");

    checkArgument(maxShardRows > 0, "maxShardRows must be > 0");
    this.maxShardRows = maxShardRows;

    this.compactionEnabled = compactionEnabled;
    this.compactionSetCreator = new CompactionSetCreator(temporalFunction, maxShardSize, maxShardRows);
}
 

@JsonProperty
public DataSize getSpilledDataSize()
{
    return succinctBytes(operatorSummaries.stream()
            .mapToLong(stats -> stats.getSpilledDataSize().toBytes())
            .sum());
}
 

/**
 * If there no data, attempt to load some from the pages supplier.
 */
private boolean loadPagesIfNecessary(PagesSupplier pagesSupplier, DataSize maxSize)
{
    checkState(!Thread.holdsLock(this), "Cannot load pages while holding a lock on this");

    boolean dataAddedOrNoMorePages;
    List<SerializedPageReference> pageReferences;
    synchronized (this) {
        if (noMorePages) {
            return false;
        }

        if (!pages.isEmpty()) {
            return false;
        }

        // The page supplier has incremented the page reference count, and addPages below also increments
        // the reference count, so we need to drop the page supplier reference. The call dereferencePage
        // is performed outside of synchronized to avoid making a callback while holding a lock.
        pageReferences = pagesSupplier.getPages(maxSize);

        // add the pages to this buffer, which will increase the reference count
        addPages(pageReferences);

        // check for no more pages
        if (!pagesSupplier.mayHaveMorePages()) {
            noMorePages = true;
        }
        dataAddedOrNoMorePages = !pageReferences.isEmpty() || noMorePages;
    }

    // sent pages will have an initial reference count, so drop it
    pageReferences.forEach(SerializedPageReference::dereferencePage);

    return dataAddedOrNoMorePages;
}
 

public ParquetReaderOptions withMaxBufferSize(DataSize maxBufferSize)
{
    return new ParquetReaderOptions(
            failOnCorruptedStatistics,
            maxReadBlockSize,
            maxMergeDistance,
            maxBufferSize);
}
 

public SortBuffer(
        DataSize maxMemory,
        List<Type> types,
        List<Integer> sortFields,
        List<SortOrder> sortOrders,
        PageSorter pageSorter)
{
    checkArgument(maxMemory.toBytes() > 0, "maxMemory is zero");
    this.maxMemoryBytes = maxMemory.toBytes();
    this.types = requireNonNull(types, "types is null");
    this.sortFields = ImmutableList.copyOf(requireNonNull(sortFields, "sortFields is null"));
    this.sortOrders = ImmutableList.copyOf(requireNonNull(sortOrders, "sortOrders is null"));
    this.pageSorter = requireNonNull(pageSorter, "pageSorter is null");
    this.pageBuilder = new PageBuilder(types);
}
 

private PartitionedOutputBuffer createPartitionedBuffer(OutputBuffers buffers, DataSize dataSize)
{
    return new PartitionedOutputBuffer(
            TASK_INSTANCE_ID,
            new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES),
            buffers,
            dataSize,
            () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"),
            stateNotificationExecutor);
}
 

public CompactionSetCreator(TemporalFunction temporalFunction, DataSize maxShardSize, long maxShardRows)
{
    checkArgument(maxShardRows > 0, "maxShardRows must be > 0");

    this.temporalFunction = requireNonNull(temporalFunction, "temporalFunction is null");
    this.maxShardSize = requireNonNull(maxShardSize, "maxShardSize is null");
    this.maxShardRows = maxShardRows;
}
 

@Override
public BasicQueryInfo getBasicQueryInfo()
{
    return new BasicQueryInfo(
            new QueryId("test"),
            session.toSessionRepresentation(),
            Optional.empty(),
            state,
            new MemoryPoolId("test"),
            !state.isDone(),
            URI.create("http://test"),
            "SELECT 1",
            Optional.empty(),
            Optional.empty(),
            new BasicQueryStats(
                    new DateTime(1),
                    new DateTime(2),
                    new Duration(3, NANOSECONDS),
                    new Duration(4, NANOSECONDS),
                    new Duration(5, NANOSECONDS),
                    6,
                    7,
                    8,
                    9,
                    DataSize.ofBytes(14),
                    15,
                    16.0,
                    memoryUsage,
                    memoryUsage,
                    DataSize.ofBytes(19),
                    DataSize.ofBytes(20),
                    cpuUsage,
                    new Duration(22, NANOSECONDS),
                    false,
                    ImmutableSet.of(),
                    OptionalDouble.empty()),
            null,
            null);
}
 

@Config("hive.text.max-line-length")
@ConfigDescription("Maximum line length for text files")
public HiveConfig setTextMaxLineLength(DataSize textMaxLineLength)
{
    this.textMaxLineLength = textMaxLineLength;
    return this;
}
 

private static OperatorFactory compileFilterWithNoInputColumns(RowExpression filter, ExpressionCompiler compiler)
{
    try {
        Supplier<PageProcessor> processor = compiler.compilePageProcessor(Optional.of(filter), ImmutableList.of());

        return FilterAndProjectOperator.createOperatorFactory(0, new PlanNodeId("test"), processor, ImmutableList.of(), DataSize.ofBytes(0), 0);
    }
    catch (Throwable e) {
        if (e instanceof UncheckedExecutionException) {
            e = e.getCause();
        }
        throw new RuntimeException("Error compiling " + filter + ": " + e.getMessage(), e);
    }
}
 

private PhysicalOperation planGroupByAggregation(
        AggregationNode node,
        PhysicalOperation source,
        boolean spillEnabled,
        DataSize unspillMemoryLimit,
        LocalExecutionPlanContext context)
{
    ImmutableMap.Builder<Symbol, Integer> mappings = ImmutableMap.builder();
    OperatorFactory operatorFactory = createHashAggregationOperatorFactory(
            node.getId(),
            node.getAggregations(),
            node.getGlobalGroupingSets(),
            node.getGroupingKeys(),
            node.getStep(),
            node.getHashSymbol(),
            node.getGroupIdSymbol(),
            source,
            node.hasDefaultOutput(),
            spillEnabled,
            node.isStreamable(),
            unspillMemoryLimit,
            context,
            0,
            mappings,
            10_000,
            Optional.of(maxPartialAggregationMemorySize),
            node.getStep().isOutputPartial());
    return new PhysicalOperation(operatorFactory, mappings.build(), context, source);
}
 

private TaskContext newTestingTaskContext(ScheduledExecutorService taskNotificationExecutor, ScheduledExecutorService driverYieldExecutor, TaskStateMachine taskStateMachine)
{
    QueryContext queryContext = new QueryContext(
            new QueryId("queryid"),
            DataSize.of(1, MEGABYTE),
            DataSize.of(2, MEGABYTE),
            new MemoryPool(new MemoryPoolId("test"), DataSize.of(1, GIGABYTE)),
            new TestingGcMonitor(),
            taskNotificationExecutor,
            driverYieldExecutor,
            DataSize.of(1, MEGABYTE),
            new SpillSpaceTracker(DataSize.of(1, GIGABYTE)));
    return queryContext.addTaskContext(taskStateMachine, TEST_SESSION, false, false, OptionalInt.empty());
}
 

@Test
public void testExplicitPropertyMappings()
{
    Map<String, String> properties = new ImmutableMap.Builder<String, String>()
            .put("thrift.client.transport", "HEADER")
            .put("thrift.client.protocol", "COMPACT")
            .put("thrift.client.connect-timeout", "99ms")
            .put("thrift.client.request-timeout", "33m")
            .put("thrift.client.socks-proxy", "localhost:11")
            .put("thrift.client.max-frame-size", "55MB")
            .put("thrift.client.ssl.enabled", "true")
            .put("thrift.client.ssl.trust-certificate", "trust")
            .put("thrift.client.ssl.key", "key")
            .put("thrift.client.ssl.key-password", "key_password")
            .put("thrift.client.ssl.session-cache-size", "678")
            .put("thrift.client.ssl.session-timeout", "78h")
            .put("thrift.client.ssl.ciphers", "some_cipher")
            .build();

    DriftNettyClientConfig expected = new DriftNettyClientConfig()
            .setTransport(HEADER)
            .setProtocol(COMPACT)
            .setConnectTimeout(new Duration(99, MILLISECONDS))
            .setRequestTimeout(new Duration(33, MINUTES))
            .setSocksProxy(HostAndPort.fromParts("localhost", 11))
            .setMaxFrameSize(new DataSize(55, MEGABYTE))
            .setSslEnabled(true)
            .setTrustCertificate(new File("trust"))
            .setKey(new File("key"))
            .setKeyPassword("key_password")
            .setSessionCacheSize(678)
            .setSessionTimeout(new Duration(78, HOURS))
            .setCiphers("some_cipher");

    assertFullMapping(properties, expected);
}
 

@Test
public void testDefaults()
{
    assertRecordedDefaults(recordDefaults(ParquetReaderConfig.class)
            .setFailOnCorruptedStatistics(true)
            .setMaxReadBlockSize(DataSize.of(16, MEGABYTE))
            .setMaxMergeDistance(DataSize.of(1, MEGABYTE))
            .setMaxBufferSize(DataSize.of(8, MEGABYTE)));
}
 

public static SqlTask createSqlTask(
        TaskId taskId,
        URI location,
        String nodeId,
        QueryContext queryContext,
        SqlTaskExecutionFactory sqlTaskExecutionFactory,
        ExecutorService taskNotificationExecutor,
        Function<SqlTask, ?> onDone,
        DataSize maxBufferSize,
        CounterStat failedTasks)
{
    SqlTask sqlTask = new SqlTask(taskId, location, nodeId, queryContext, sqlTaskExecutionFactory, taskNotificationExecutor, maxBufferSize);
    sqlTask.initialize(onDone, failedTasks);
    return sqlTask;
}