Java Code Examples for io.prestosql.spi.predicate.TupleDomain#isNone()

private List<String> toConjuncts(
        JdbcClient client,
        ConnectorSession session,
        Connection connection,
        TupleDomain<ColumnHandle> tupleDomain,
        List<TypeAndValue> accumulator)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of(ALWAYS_FALSE);
    }
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
        JdbcColumnHandle column = ((JdbcColumnHandle) entry.getKey());
        Domain domain = pushDownDomain(client, session, connection, column, entry.getValue());
        builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
    }
    return builder.build();
}
 

private Optional<Set<String>> calculateGrantees(
        ConnectorSession connectorSession,
        TupleDomain<ColumnHandle> constraint,
        Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Optional<Set<String>> grantees = filterString(constraint, GRANTEE_COLUMN_HANDLE);
    if (grantees.isEmpty()) {
        return Optional.empty();
    }

    Set<String> result = grantees.get().stream()
            .filter(this::isLowerCase)
            .filter(role -> !predicate.isPresent() || predicate.get().test(granteeAsFixedValues(role)))
            .collect(toImmutableSet());

    if (!result.isEmpty() && result.size() <= MAX_ROLE_COUNT) {
        return Optional.of(result);
    }

    return Optional.empty();
}
 

private List<CassandraPartition> getCassandraPartitions(CassandraTable table, TupleDomain<ColumnHandle> tupleDomain)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of();
    }

    List<Set<Object>> partitionKeysList = getPartitionKeysList(table, tupleDomain);

    Set<List<Object>> filterList = Sets.cartesianProduct(partitionKeysList);
    // empty filters means, all partitions
    if (filterList.isEmpty()) {
        return cassandraSession.getPartitions(table, ImmutableList.of());
    }

    return cassandraSession.getPartitions(table, partitionKeysList);
}
 

private static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<IcebergColumnHandle> effectivePredicate)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    effectivePredicate.getDomains().get().forEach((columnHandle, domain) -> {
        String baseType = columnHandle.getType().getTypeSignature().getBase();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (!baseType.equals(StandardTypes.MAP) && !baseType.equals(StandardTypes.ARRAY) && !baseType.equals(StandardTypes.ROW)) {
            RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
            if (descriptor != null) {
                predicate.put(descriptor, domain);
            }
        }
    });
    return TupleDomain.withColumnDomains(predicate.build());
}
 

private Map<TpchColumn<?>, List<Object>> getColumnValuesRestrictions(TpchTable<?> tpchTable, Constraint constraint)
{
    TupleDomain<ColumnHandle> constraintSummary = constraint.getSummary();
    if (constraintSummary.isAll()) {
        return emptyMap();
    }
    else if (constraintSummary.isNone()) {
        Set<TpchColumn<?>> columns = ImmutableSet.copyOf(tpchTable.getColumns());
        return asMap(columns, key -> emptyList());
    }
    else {
        Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get();
        Optional<Domain> orderStatusDomain = Optional.ofNullable(domains.get(toColumnHandle(ORDER_STATUS)));
        Optional<Map<TpchColumn<?>, List<Object>>> allowedColumnValues = orderStatusDomain.map(domain -> {
            List<Object> allowedValues = ORDER_STATUS_VALUES.stream()
                    .filter(domain::includesNullableValue)
                    .collect(toList());
            return avoidTrivialOrderStatusRestriction(allowedValues);
        });
        return allowedColumnValues.orElse(emptyMap());
    }
}
 

@Override
public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List<ColumnHandle> columns, Supplier<TupleDomain<ColumnHandle>> dynamicFilter)
{
    requireNonNull(columns, "columns is null");
    checkArgument(split.getCatalogName().equals(table.getCatalogName()), "mismatched split and table");
    CatalogName catalogName = split.getCatalogName();

    ConnectorPageSourceProvider provider = getPageSourceProvider(catalogName);
    TupleDomain<ColumnHandle> constraint = dynamicFilter.get();
    if (constraint.isNone()) {
        return new EmptyPageSource();
    }
    return provider.createPageSource(
            table.getTransaction(),
            session.toConnectorSession(catalogName),
            split.getConnectorSplit(),
            table.getConnectorHandle(),
            columns,
            constraint);
}
 

private Optional<Set<String>> calculateRoles(
        ConnectorSession connectorSession,
        TupleDomain<ColumnHandle> constraint,
        Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Optional<Set<String>> roles = filterString(constraint, ROLE_NAME_COLUMN_HANDLE);
    if (roles.isPresent()) {
        Set<String> result = roles.get().stream()
                .filter(this::isLowerCase)
                .filter(role -> !predicate.isPresent() || predicate.get().test(roleAsFixedValues(role)))
                .collect(toImmutableSet());

        if (result.isEmpty()) {
            return Optional.empty();
        }
        if (result.size() <= MAX_ROLE_COUNT) {
            return Optional.of(result);
        }
    }

    if (predicate.isEmpty()) {
        return Optional.empty();
    }

    Session session = ((FullConnectorSession) connectorSession).getSession();
    return Optional.of(metadata.listRoles(session, catalogName)
            .stream()
            .filter(role -> predicate.get().test(roleAsFixedValues(role)))
            .collect(toImmutableSet()));
}
 

public static TupleDomain<ColumnDescriptor> getParquetTupleDomain(
        Map<List<String>, RichColumnDescriptor> descriptorsByPath,
        TupleDomain<HiveColumnHandle> effectivePredicate,
        MessageType fileSchema,
        boolean useColumnNames)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    for (Entry<HiveColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) {
        HiveColumnHandle columnHandle = entry.getKey();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (columnHandle.getHiveType().getCategory() != PRIMITIVE || columnHandle.getColumnType() != REGULAR) {
            continue;
        }

        RichColumnDescriptor descriptor;
        if (useColumnNames) {
            descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
        }
        else {
            org.apache.parquet.schema.Type parquetField = getParquetType(columnHandle, fileSchema, false);
            if (parquetField == null || !parquetField.isPrimitive()) {
                // Parquet file has fewer column than partition
                // Or the field is a complex type
                continue;
            }
            descriptor = descriptorsByPath.get(ImmutableList.of(parquetField.getName()));
        }
        if (descriptor != null) {
            predicate.put(descriptor, entry.getValue());
        }
    }
    return TupleDomain.withColumnDomains(predicate.build());
}
 

@SuppressWarnings("OptionalGetWithoutIsPresent")
private static String getWhereClause(
        TupleDomain<Integer> tupleDomain,
        List<String> columnNames,
        List<Type> types,
        Set<Integer> uuidColumnIndexes,
        List<ValueBuffer> bindValues)
{
    if (tupleDomain.isNone()) {
        return "";
    }

    ImmutableList.Builder<String> conjunctsBuilder = ImmutableList.builder();
    Map<Integer, Domain> domainMap = tupleDomain.getDomains().get();
    for (Map.Entry<Integer, Domain> entry : domainMap.entrySet()) {
        int index = entry.getKey();
        String columnName = columnNames.get(index);
        Type type = types.get(index);
        conjunctsBuilder.add(toPredicate(index, columnName, type, entry.getValue(), uuidColumnIndexes, bindValues));
    }
    List<String> conjuncts = conjunctsBuilder.build();

    if (conjuncts.isEmpty()) {
        return "";
    }
    StringBuilder where = new StringBuilder("WHERE ");
    return Joiner.on(" AND\n").appendTo(where, conjuncts).toString();
}
 

@Override
public ConnectorPageSource createPageSource(
        ConnectorTransactionHandle transaction,
        ConnectorSession session,
        ConnectorSplit split,
        ConnectorTableHandle table,
        List<ColumnHandle> columns,
        TupleDomain<ColumnHandle> dynamicFilter)
{
    MemorySplit memorySplit = (MemorySplit) split;
    long tableId = memorySplit.getTable();
    int partNumber = memorySplit.getPartNumber();
    int totalParts = memorySplit.getTotalPartsPerWorker();
    long expectedRows = memorySplit.getExpectedRows();
    MemoryTableHandle memoryTable = (MemoryTableHandle) table;
    OptionalDouble sampleRatio = memoryTable.getSampleRatio();

    if (dynamicFilter.isNone()) {
        return new FixedPageSource(ImmutableList.of());
    }
    Map<Integer, Domain> domains = dynamicFilter
            .transform(columns::indexOf)
            .getDomains()
            .get();

    List<Integer> columnIndexes = columns.stream()
            .map(MemoryColumnHandle.class::cast)
            .map(MemoryColumnHandle::getColumnIndex).collect(toList());
    List<Page> pages = pagesStore.getPages(
            tableId,
            partNumber,
            totalParts,
            columnIndexes,
            expectedRows,
            memorySplit.getLimit(),
            sampleRatio);
    return new FixedPageSource(pages.stream()
            .map(page -> applyFilter(page, domains))
            .collect(toList()));
}
 

public static boolean partitionMatches(List<HiveColumnHandle> partitionColumns, TupleDomain<ColumnHandle> constraintSummary, HivePartition partition)
{
    if (constraintSummary.isNone()) {
        return false;
    }
    Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get();
    for (HiveColumnHandle column : partitionColumns) {
        NullableValue value = partition.getKeys().get(column);
        Domain allowedDomain = domains.get(column);
        if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) {
            return false;
        }
    }
    return true;
}
 

public Expression toPredicate(TupleDomain<Symbol> tupleDomain)
{
    if (tupleDomain.isNone()) {
        return FALSE_LITERAL;
    }

    Map<Symbol, Domain> domains = tupleDomain.getDomains().get();
    return domains.entrySet().stream()
            .map(entry -> toPredicate(entry.getValue(), entry.getKey().toSymbolReference()))
            .collect(collectingAndThen(toImmutableList(), expressions -> ExpressionUtils.combineConjuncts(metadata, expressions)));
}
 

private static OptionalInt extractUpperBound(TupleDomain<Symbol> tupleDomain, Symbol symbol)
{
    if (tupleDomain.isNone()) {
        return OptionalInt.empty();
    }

    Domain rowNumberDomain = tupleDomain.getDomains().get().get(symbol);
    if (rowNumberDomain == null) {
        return OptionalInt.empty();
    }
    ValueSet values = rowNumberDomain.getValues();
    if (values.isAll() || values.isNone() || values.getRanges().getRangeCount() <= 0) {
        return OptionalInt.empty();
    }

    Range span = values.getRanges().getSpan();

    if (span.getHigh().isUpperUnbounded()) {
        return OptionalInt.empty();
    }

    verify(rowNumberDomain.getType().equals(BIGINT));
    long upperBound = (Long) span.getHigh().getValue();
    if (span.getHigh().getBound() == BELOW) {
        upperBound--;
    }

    if (upperBound >= Integer.MIN_VALUE && upperBound <= Integer.MAX_VALUE) {
        return OptionalInt.of(toIntExact(upperBound));
    }
    return OptionalInt.empty();
}
 

private static boolean allRowNumberValuesInDomain(TupleDomain<Symbol> tupleDomain, Symbol symbol, long upperBound)
{
    if (tupleDomain.isNone()) {
        return false;
    }
    Domain domain = tupleDomain.getDomains().get().get(symbol);
    if (domain == null) {
        return true;
    }
    return domain.getValues().contains(ValueSet.ofRanges(range(domain.getType(), 0L, true, upperBound, true)));
}
 

private Map<DynamicFilterId, Domain> convertTupleDomain(TupleDomain<DynamicFilterId> result)
{
    if (result.isNone()) {
        // One of the join build symbols has no non-null values, therefore no filters can match predicate
        return buildChannels.keySet().stream()
                .collect(toImmutableMap(identity(), filterId -> Domain.none(filterBuildTypes.get(filterId))));
    }
    return result.getDomains().get();
}
 

public static <T> Optional<String> tryGetSingleVarcharValue(TupleDomain<T> constraint, T index)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Domain domain = constraint.getDomains().get().get(index);
    if ((domain == null) || !domain.isSingleValue()) {
        return Optional.empty();
    }

    Object value = domain.getSingleValue();
    return Optional.of(((Slice) value).toStringUtf8());
}
 

@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint)
{
    SystemTableHandle table = (SystemTableHandle) handle;

    TupleDomain<ColumnHandle> oldDomain = table.getConstraint();
    TupleDomain<ColumnHandle> newDomain = oldDomain.intersect(constraint.getSummary());
    if (oldDomain.equals(newDomain) && constraint.predicate().isEmpty()) {
        return Optional.empty();
    }

    SystemTable systemTable = checkAndGetTable(session, table);
    if (systemTable instanceof JdbcTable) {
        TupleDomain<ColumnHandle> filtered = ((JdbcTable) systemTable).applyFilter(session, new Constraint(newDomain, constraint.predicate(), constraint.getColumns()));
        newDomain = newDomain.intersect(filtered);
    }

    if (oldDomain.equals(newDomain)) {
        return Optional.empty();
    }

    if (newDomain.isNone()) {
        // TODO (https://github.com/prestosql/presto/issues/3647) indicate the table scan is empty
    }
    table = new SystemTableHandle(table.getSchemaName(), table.getTableName(), newDomain);
    return Optional.of(new ConstraintApplicationResult<>(table, constraint.getSummary()));
}
 

private <T> Optional<Set<String>> filterString(TupleDomain<T> constraint, T column)
{
    if (constraint.isNone()) {
        return Optional.of(ImmutableSet.of());
    }

    Domain domain = constraint.getDomains().get().get(column);
    if (domain == null) {
        return Optional.empty();
    }

    if (domain.isSingleValue()) {
        return Optional.of(ImmutableSet.of(((Slice) domain.getSingleValue()).toStringUtf8()));
    }
    if (domain.getValues() instanceof EquatableValueSet) {
        Collection<Object> values = ((EquatableValueSet) domain.getValues()).getValues();
        return Optional.of(values.stream()
                .map(Slice.class::cast)
                .map(Slice::toStringUtf8)
                .collect(toImmutableSet()));
    }
    if (domain.getValues() instanceof SortedRangeSet) {
        ImmutableSet.Builder<String> result = ImmutableSet.builder();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            checkState(!range.isAll()); // Already checked
            if (!range.isSingleValue()) {
                return Optional.empty();
            }

            result.add(((Slice) range.getSingleValue()).toStringUtf8());
        }

        return Optional.of(result.build());
    }
    return Optional.empty();
}
 

public static TupleDomain<ColumnHandle> computeEnforced(TupleDomain<ColumnHandle> predicate, TupleDomain<ColumnHandle> unenforced)
{
    if (predicate.isNone()) {
        // If the engine requests that the connector provides a layout with a domain of "none". The connector can have two possible reactions, either:
        // 1. The connector can provide an empty table layout.
        //   * There would be no unenforced predicate, i.e., unenforced predicate is TupleDomain.all().
        //   * The predicate was successfully enforced. Enforced predicate would be same as predicate: TupleDomain.none().
        // 2. The connector can't/won't.
        //   * The connector would tell the engine to put a filter on top of the scan, i.e., unenforced predicate is TupleDomain.none().
        //   * The connector didn't successfully enforce anything. Therefore, enforced predicate would be TupleDomain.all().
        if (unenforced.isNone()) {
            return TupleDomain.all();
        }
        if (unenforced.isAll()) {
            return TupleDomain.none();
        }
        throw new IllegalArgumentException();
    }

    // The engine requested the connector provides a layout with a non-none TupleDomain.
    // A TupleDomain is effectively a list of column-Domain pairs.
    // The connector is expected enforce the respective domain entirely on none, some, or all of the columns.
    // 1. When the connector could enforce none of the domains, the unenforced would be equal to predicate;
    // 2. When the connector could enforce some of the domains, the unenforced would contain a subset of the column-Domain pairs;
    // 3. When the connector could enforce all of the domains, the unenforced would be TupleDomain.all().

    // In all 3 cases shown above, the unenforced is not TupleDomain.none().
    checkArgument(!unenforced.isNone());

    Map<ColumnHandle, Domain> predicateDomains = predicate.getDomains().get();
    Map<ColumnHandle, Domain> unenforcedDomains = unenforced.getDomains().get();
    ImmutableMap.Builder<ColumnHandle, Domain> enforcedDomainsBuilder = ImmutableMap.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : predicateDomains.entrySet()) {
        ColumnHandle predicateColumnHandle = entry.getKey();
        if (unenforcedDomains.containsKey(predicateColumnHandle)) {
            checkArgument(
                    entry.getValue().equals(unenforcedDomains.get(predicateColumnHandle)),
                    "Enforced tuple domain cannot be determined. The connector is expected to enforce the respective domain entirely on none, some, or all of the column.");
        }
        else {
            enforcedDomainsBuilder.put(predicateColumnHandle, entry.getValue());
        }
    }
    Map<ColumnHandle, Domain> enforcedDomains = enforcedDomainsBuilder.build();
    checkArgument(
            enforcedDomains.size() + unenforcedDomains.size() == predicateDomains.size(),
            "Enforced tuple domain cannot be determined. Connector returned an unenforced TupleDomain that contains columns not in predicate.");
    return TupleDomain.withColumnDomains(enforcedDomains);
}
 

public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastore, HiveIdentity identity, ConnectorTableHandle tableHandle, Constraint constraint)
{
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    TupleDomain<ColumnHandle> effectivePredicate = constraint.getSummary()
            .intersect(hiveTableHandle.getEnforcedConstraint());

    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    Optional<HiveBucketHandle> hiveBucketHandle = hiveTableHandle.getBucketHandle();
    List<HiveColumnHandle> partitionColumns = hiveTableHandle.getPartitionColumns();

    if (effectivePredicate.isNone()) {
        return new HivePartitionResult(partitionColumns, ImmutableList.of(), none(), none(), none(), hiveBucketHandle, Optional.empty());
    }

    Table table = metastore.getTable(identity, tableName.getSchemaName(), tableName.getTableName())
            .orElseThrow(() -> new TableNotFoundException(tableName));

    Optional<HiveBucketFilter> bucketFilter = getHiveBucketFilter(table, effectivePredicate);
    TupleDomain<HiveColumnHandle> compactEffectivePredicate = effectivePredicate
            .transform(HiveColumnHandle.class::cast)
            .simplify(domainCompactionThreshold);

    if (partitionColumns.isEmpty()) {
        return new HivePartitionResult(
                partitionColumns,
                ImmutableList.of(new HivePartition(tableName)),
                compactEffectivePredicate,
                effectivePredicate,
                TupleDomain.all(),
                hiveBucketHandle,
                bucketFilter);
    }

    List<Type> partitionTypes = partitionColumns.stream()
            .map(HiveColumnHandle::getType)
            .collect(toList());

    Iterable<HivePartition> partitionsIterable;
    Predicate<Map<ColumnHandle, NullableValue>> predicate = constraint.predicate().orElse(value -> true);
    if (hiveTableHandle.getPartitions().isPresent()) {
        partitionsIterable = hiveTableHandle.getPartitions().get().stream()
                .filter(partition -> partitionMatches(partitionColumns, effectivePredicate, predicate, partition))
                .collect(toImmutableList());
    }
    else {
        List<String> partitionNames = getFilteredPartitionNames(metastore, identity, tableName, partitionColumns, effectivePredicate);
        partitionsIterable = () -> partitionNames.stream()
                // Apply extra filters which could not be done by getFilteredPartitionNames
                .map(partitionName -> parseValuesAndFilterPartition(tableName, partitionName, partitionColumns, partitionTypes, effectivePredicate, predicate))
                .filter(Optional::isPresent)
                .map(Optional::get)
                .iterator();
    }

    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = effectivePredicate.filter((column, domain) -> !partitionColumns.contains(column));
    TupleDomain<ColumnHandle> enforcedTupleDomain = effectivePredicate.filter((column, domain) -> partitionColumns.contains(column));
    return new HivePartitionResult(partitionColumns, partitionsIterable, compactEffectivePredicate, remainingTupleDomain, enforcedTupleDomain, hiveBucketHandle, bucketFilter);
}