Java Code Examples for io.prestosql.spi.predicate.TupleDomain#isNone()

The following examples show how to use io.prestosql.spi.predicate.TupleDomain#isNone() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: QueryBuilder.java    From presto with Apache License 2.0 6 votes vote down vote up
private List<String> toConjuncts(
        JdbcClient client,
        ConnectorSession session,
        Connection connection,
        TupleDomain<ColumnHandle> tupleDomain,
        List<TypeAndValue> accumulator)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of(ALWAYS_FALSE);
    }
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
        JdbcColumnHandle column = ((JdbcColumnHandle) entry.getKey());
        Domain domain = pushDownDomain(client, session, connection, column, entry.getValue());
        builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
    }
    return builder.build();
}
 
Example 2
Source File: InformationSchemaMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
private Optional<Set<String>> calculateGrantees(
        ConnectorSession connectorSession,
        TupleDomain<ColumnHandle> constraint,
        Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Optional<Set<String>> grantees = filterString(constraint, GRANTEE_COLUMN_HANDLE);
    if (grantees.isEmpty()) {
        return Optional.empty();
    }

    Set<String> result = grantees.get().stream()
            .filter(this::isLowerCase)
            .filter(role -> !predicate.isPresent() || predicate.get().test(granteeAsFixedValues(role)))
            .collect(toImmutableSet());

    if (!result.isEmpty() && result.size() <= MAX_ROLE_COUNT) {
        return Optional.of(result);
    }

    return Optional.empty();
}
 
Example 3
Source File: CassandraPartitionManager.java    From presto with Apache License 2.0 6 votes vote down vote up
private List<CassandraPartition> getCassandraPartitions(CassandraTable table, TupleDomain<ColumnHandle> tupleDomain)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of();
    }

    List<Set<Object>> partitionKeysList = getPartitionKeysList(table, tupleDomain);

    Set<List<Object>> filterList = Sets.cartesianProduct(partitionKeysList);
    // empty filters means, all partitions
    if (filterList.isEmpty()) {
        return cassandraSession.getPartitions(table, ImmutableList.of());
    }

    return cassandraSession.getPartitions(table, partitionKeysList);
}
 
Example 4
Source File: IcebergPageSourceProvider.java    From presto with Apache License 2.0 6 votes vote down vote up
private static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<IcebergColumnHandle> effectivePredicate)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    effectivePredicate.getDomains().get().forEach((columnHandle, domain) -> {
        String baseType = columnHandle.getType().getTypeSignature().getBase();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (!baseType.equals(StandardTypes.MAP) && !baseType.equals(StandardTypes.ARRAY) && !baseType.equals(StandardTypes.ROW)) {
            RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
            if (descriptor != null) {
                predicate.put(descriptor, domain);
            }
        }
    });
    return TupleDomain.withColumnDomains(predicate.build());
}
 
Example 5
Source File: TpchMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
private Map<TpchColumn<?>, List<Object>> getColumnValuesRestrictions(TpchTable<?> tpchTable, Constraint constraint)
{
    TupleDomain<ColumnHandle> constraintSummary = constraint.getSummary();
    if (constraintSummary.isAll()) {
        return emptyMap();
    }
    else if (constraintSummary.isNone()) {
        Set<TpchColumn<?>> columns = ImmutableSet.copyOf(tpchTable.getColumns());
        return asMap(columns, key -> emptyList());
    }
    else {
        Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get();
        Optional<Domain> orderStatusDomain = Optional.ofNullable(domains.get(toColumnHandle(ORDER_STATUS)));
        Optional<Map<TpchColumn<?>, List<Object>>> allowedColumnValues = orderStatusDomain.map(domain -> {
            List<Object> allowedValues = ORDER_STATUS_VALUES.stream()
                    .filter(domain::includesNullableValue)
                    .collect(toList());
            return avoidTrivialOrderStatusRestriction(allowedValues);
        });
        return allowedColumnValues.orElse(emptyMap());
    }
}
 
Example 6
Source File: PageSourceManager.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public ConnectorPageSource createPageSource(Session session, Split split, TableHandle table, List<ColumnHandle> columns, Supplier<TupleDomain<ColumnHandle>> dynamicFilter)
{
    requireNonNull(columns, "columns is null");
    checkArgument(split.getCatalogName().equals(table.getCatalogName()), "mismatched split and table");
    CatalogName catalogName = split.getCatalogName();

    ConnectorPageSourceProvider provider = getPageSourceProvider(catalogName);
    TupleDomain<ColumnHandle> constraint = dynamicFilter.get();
    if (constraint.isNone()) {
        return new EmptyPageSource();
    }
    return provider.createPageSource(
            table.getTransaction(),
            session.toConnectorSession(catalogName),
            split.getConnectorSplit(),
            table.getConnectorHandle(),
            columns,
            constraint);
}
 
Example 7
Source File: InformationSchemaMetadata.java    From presto with Apache License 2.0 5 votes vote down vote up
private Optional<Set<String>> calculateRoles(
        ConnectorSession connectorSession,
        TupleDomain<ColumnHandle> constraint,
        Optional<Predicate<Map<ColumnHandle, NullableValue>>> predicate)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Optional<Set<String>> roles = filterString(constraint, ROLE_NAME_COLUMN_HANDLE);
    if (roles.isPresent()) {
        Set<String> result = roles.get().stream()
                .filter(this::isLowerCase)
                .filter(role -> !predicate.isPresent() || predicate.get().test(roleAsFixedValues(role)))
                .collect(toImmutableSet());

        if (result.isEmpty()) {
            return Optional.empty();
        }
        if (result.size() <= MAX_ROLE_COUNT) {
            return Optional.of(result);
        }
    }

    if (predicate.isEmpty()) {
        return Optional.empty();
    }

    Session session = ((FullConnectorSession) connectorSession).getSession();
    return Optional.of(metadata.listRoles(session, catalogName)
            .stream()
            .filter(role -> predicate.get().test(roleAsFixedValues(role)))
            .collect(toImmutableSet()));
}
 
Example 8
Source File: ParquetPageSourceFactory.java    From presto with Apache License 2.0 5 votes vote down vote up
public static TupleDomain<ColumnDescriptor> getParquetTupleDomain(
        Map<List<String>, RichColumnDescriptor> descriptorsByPath,
        TupleDomain<HiveColumnHandle> effectivePredicate,
        MessageType fileSchema,
        boolean useColumnNames)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    for (Entry<HiveColumnHandle, Domain> entry : effectivePredicate.getDomains().get().entrySet()) {
        HiveColumnHandle columnHandle = entry.getKey();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (columnHandle.getHiveType().getCategory() != PRIMITIVE || columnHandle.getColumnType() != REGULAR) {
            continue;
        }

        RichColumnDescriptor descriptor;
        if (useColumnNames) {
            descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
        }
        else {
            org.apache.parquet.schema.Type parquetField = getParquetType(columnHandle, fileSchema, false);
            if (parquetField == null || !parquetField.isPrimitive()) {
                // Parquet file has fewer column than partition
                // Or the field is a complex type
                continue;
            }
            descriptor = descriptorsByPath.get(ImmutableList.of(parquetField.getName()));
        }
        if (descriptor != null) {
            predicate.put(descriptor, entry.getValue());
        }
    }
    return TupleDomain.withColumnDomains(predicate.build());
}
 
Example 9
Source File: PreparedStatementBuilder.java    From presto with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("OptionalGetWithoutIsPresent")
private static String getWhereClause(
        TupleDomain<Integer> tupleDomain,
        List<String> columnNames,
        List<Type> types,
        Set<Integer> uuidColumnIndexes,
        List<ValueBuffer> bindValues)
{
    if (tupleDomain.isNone()) {
        return "";
    }

    ImmutableList.Builder<String> conjunctsBuilder = ImmutableList.builder();
    Map<Integer, Domain> domainMap = tupleDomain.getDomains().get();
    for (Map.Entry<Integer, Domain> entry : domainMap.entrySet()) {
        int index = entry.getKey();
        String columnName = columnNames.get(index);
        Type type = types.get(index);
        conjunctsBuilder.add(toPredicate(index, columnName, type, entry.getValue(), uuidColumnIndexes, bindValues));
    }
    List<String> conjuncts = conjunctsBuilder.build();

    if (conjuncts.isEmpty()) {
        return "";
    }
    StringBuilder where = new StringBuilder("WHERE ");
    return Joiner.on(" AND\n").appendTo(where, conjuncts).toString();
}
 
Example 10
Source File: MemoryPageSourceProvider.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public ConnectorPageSource createPageSource(
        ConnectorTransactionHandle transaction,
        ConnectorSession session,
        ConnectorSplit split,
        ConnectorTableHandle table,
        List<ColumnHandle> columns,
        TupleDomain<ColumnHandle> dynamicFilter)
{
    MemorySplit memorySplit = (MemorySplit) split;
    long tableId = memorySplit.getTable();
    int partNumber = memorySplit.getPartNumber();
    int totalParts = memorySplit.getTotalPartsPerWorker();
    long expectedRows = memorySplit.getExpectedRows();
    MemoryTableHandle memoryTable = (MemoryTableHandle) table;
    OptionalDouble sampleRatio = memoryTable.getSampleRatio();

    if (dynamicFilter.isNone()) {
        return new FixedPageSource(ImmutableList.of());
    }
    Map<Integer, Domain> domains = dynamicFilter
            .transform(columns::indexOf)
            .getDomains()
            .get();

    List<Integer> columnIndexes = columns.stream()
            .map(MemoryColumnHandle.class::cast)
            .map(MemoryColumnHandle::getColumnIndex).collect(toList());
    List<Page> pages = pagesStore.getPages(
            tableId,
            partNumber,
            totalParts,
            columnIndexes,
            expectedRows,
            memorySplit.getLimit(),
            sampleRatio);
    return new FixedPageSource(pages.stream()
            .map(page -> applyFilter(page, domains))
            .collect(toList()));
}
 
Example 11
Source File: HivePartitionManager.java    From presto with Apache License 2.0 5 votes vote down vote up
public static boolean partitionMatches(List<HiveColumnHandle> partitionColumns, TupleDomain<ColumnHandle> constraintSummary, HivePartition partition)
{
    if (constraintSummary.isNone()) {
        return false;
    }
    Map<ColumnHandle, Domain> domains = constraintSummary.getDomains().get();
    for (HiveColumnHandle column : partitionColumns) {
        NullableValue value = partition.getKeys().get(column);
        Domain allowedDomain = domains.get(column);
        if (allowedDomain != null && !allowedDomain.includesNullableValue(value.getValue())) {
            return false;
        }
    }
    return true;
}
 
Example 12
Source File: DomainTranslator.java    From presto with Apache License 2.0 5 votes vote down vote up
public Expression toPredicate(TupleDomain<Symbol> tupleDomain)
{
    if (tupleDomain.isNone()) {
        return FALSE_LITERAL;
    }

    Map<Symbol, Domain> domains = tupleDomain.getDomains().get();
    return domains.entrySet().stream()
            .map(entry -> toPredicate(entry.getValue(), entry.getKey().toSymbolReference()))
            .collect(collectingAndThen(toImmutableList(), expressions -> ExpressionUtils.combineConjuncts(metadata, expressions)));
}
 
Example 13
Source File: WindowFilterPushDown.java    From presto with Apache License 2.0 5 votes vote down vote up
private static OptionalInt extractUpperBound(TupleDomain<Symbol> tupleDomain, Symbol symbol)
{
    if (tupleDomain.isNone()) {
        return OptionalInt.empty();
    }

    Domain rowNumberDomain = tupleDomain.getDomains().get().get(symbol);
    if (rowNumberDomain == null) {
        return OptionalInt.empty();
    }
    ValueSet values = rowNumberDomain.getValues();
    if (values.isAll() || values.isNone() || values.getRanges().getRangeCount() <= 0) {
        return OptionalInt.empty();
    }

    Range span = values.getRanges().getSpan();

    if (span.getHigh().isUpperUnbounded()) {
        return OptionalInt.empty();
    }

    verify(rowNumberDomain.getType().equals(BIGINT));
    long upperBound = (Long) span.getHigh().getValue();
    if (span.getHigh().getBound() == BELOW) {
        upperBound--;
    }

    if (upperBound >= Integer.MIN_VALUE && upperBound <= Integer.MAX_VALUE) {
        return OptionalInt.of(toIntExact(upperBound));
    }
    return OptionalInt.empty();
}
 
Example 14
Source File: WindowFilterPushDown.java    From presto with Apache License 2.0 5 votes vote down vote up
private static boolean allRowNumberValuesInDomain(TupleDomain<Symbol> tupleDomain, Symbol symbol, long upperBound)
{
    if (tupleDomain.isNone()) {
        return false;
    }
    Domain domain = tupleDomain.getDomains().get().get(symbol);
    if (domain == null) {
        return true;
    }
    return domain.getValues().contains(ValueSet.ofRanges(range(domain.getType(), 0L, true, upperBound, true)));
}
 
Example 15
Source File: LocalDynamicFilterConsumer.java    From presto with Apache License 2.0 5 votes vote down vote up
private Map<DynamicFilterId, Domain> convertTupleDomain(TupleDomain<DynamicFilterId> result)
{
    if (result.isNone()) {
        // One of the join build symbols has no non-null values, therefore no filters can match predicate
        return buildChannels.keySet().stream()
                .collect(toImmutableMap(identity(), filterId -> Domain.none(filterBuildTypes.get(filterId))));
    }
    return result.getDomains().get();
}
 
Example 16
Source File: FilterUtil.java    From presto with Apache License 2.0 5 votes vote down vote up
public static <T> Optional<String> tryGetSingleVarcharValue(TupleDomain<T> constraint, T index)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Domain domain = constraint.getDomains().get().get(index);
    if ((domain == null) || !domain.isSingleValue()) {
        return Optional.empty();
    }

    Object value = domain.getSingleValue();
    return Optional.of(((Slice) value).toStringUtf8());
}
 
Example 17
Source File: SystemTablesMetadata.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public Optional<ConstraintApplicationResult<ConnectorTableHandle>> applyFilter(ConnectorSession session, ConnectorTableHandle handle, Constraint constraint)
{
    SystemTableHandle table = (SystemTableHandle) handle;

    TupleDomain<ColumnHandle> oldDomain = table.getConstraint();
    TupleDomain<ColumnHandle> newDomain = oldDomain.intersect(constraint.getSummary());
    if (oldDomain.equals(newDomain) && constraint.predicate().isEmpty()) {
        return Optional.empty();
    }

    SystemTable systemTable = checkAndGetTable(session, table);
    if (systemTable instanceof JdbcTable) {
        TupleDomain<ColumnHandle> filtered = ((JdbcTable) systemTable).applyFilter(session, new Constraint(newDomain, constraint.predicate(), constraint.getColumns()));
        newDomain = newDomain.intersect(filtered);
    }

    if (oldDomain.equals(newDomain)) {
        return Optional.empty();
    }

    if (newDomain.isNone()) {
        // TODO (https://github.com/prestosql/presto/issues/3647) indicate the table scan is empty
    }
    table = new SystemTableHandle(table.getSchemaName(), table.getTableName(), newDomain);
    return Optional.of(new ConstraintApplicationResult<>(table, constraint.getSummary()));
}
 
Example 18
Source File: InformationSchemaMetadata.java    From presto with Apache License 2.0 5 votes vote down vote up
private <T> Optional<Set<String>> filterString(TupleDomain<T> constraint, T column)
{
    if (constraint.isNone()) {
        return Optional.of(ImmutableSet.of());
    }

    Domain domain = constraint.getDomains().get().get(column);
    if (domain == null) {
        return Optional.empty();
    }

    if (domain.isSingleValue()) {
        return Optional.of(ImmutableSet.of(((Slice) domain.getSingleValue()).toStringUtf8()));
    }
    if (domain.getValues() instanceof EquatableValueSet) {
        Collection<Object> values = ((EquatableValueSet) domain.getValues()).getValues();
        return Optional.of(values.stream()
                .map(Slice.class::cast)
                .map(Slice::toStringUtf8)
                .collect(toImmutableSet()));
    }
    if (domain.getValues() instanceof SortedRangeSet) {
        ImmutableSet.Builder<String> result = ImmutableSet.builder();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            checkState(!range.isAll()); // Already checked
            if (!range.isSingleValue()) {
                return Optional.empty();
            }

            result.add(((Slice) range.getSingleValue()).toStringUtf8());
        }

        return Optional.of(result.build());
    }
    return Optional.empty();
}
 
Example 19
Source File: TableLayoutResult.java    From presto with Apache License 2.0 4 votes vote down vote up
public static TupleDomain<ColumnHandle> computeEnforced(TupleDomain<ColumnHandle> predicate, TupleDomain<ColumnHandle> unenforced)
{
    if (predicate.isNone()) {
        // If the engine requests that the connector provides a layout with a domain of "none". The connector can have two possible reactions, either:
        // 1. The connector can provide an empty table layout.
        //   * There would be no unenforced predicate, i.e., unenforced predicate is TupleDomain.all().
        //   * The predicate was successfully enforced. Enforced predicate would be same as predicate: TupleDomain.none().
        // 2. The connector can't/won't.
        //   * The connector would tell the engine to put a filter on top of the scan, i.e., unenforced predicate is TupleDomain.none().
        //   * The connector didn't successfully enforce anything. Therefore, enforced predicate would be TupleDomain.all().
        if (unenforced.isNone()) {
            return TupleDomain.all();
        }
        if (unenforced.isAll()) {
            return TupleDomain.none();
        }
        throw new IllegalArgumentException();
    }

    // The engine requested the connector provides a layout with a non-none TupleDomain.
    // A TupleDomain is effectively a list of column-Domain pairs.
    // The connector is expected enforce the respective domain entirely on none, some, or all of the columns.
    // 1. When the connector could enforce none of the domains, the unenforced would be equal to predicate;
    // 2. When the connector could enforce some of the domains, the unenforced would contain a subset of the column-Domain pairs;
    // 3. When the connector could enforce all of the domains, the unenforced would be TupleDomain.all().

    // In all 3 cases shown above, the unenforced is not TupleDomain.none().
    checkArgument(!unenforced.isNone());

    Map<ColumnHandle, Domain> predicateDomains = predicate.getDomains().get();
    Map<ColumnHandle, Domain> unenforcedDomains = unenforced.getDomains().get();
    ImmutableMap.Builder<ColumnHandle, Domain> enforcedDomainsBuilder = ImmutableMap.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : predicateDomains.entrySet()) {
        ColumnHandle predicateColumnHandle = entry.getKey();
        if (unenforcedDomains.containsKey(predicateColumnHandle)) {
            checkArgument(
                    entry.getValue().equals(unenforcedDomains.get(predicateColumnHandle)),
                    "Enforced tuple domain cannot be determined. The connector is expected to enforce the respective domain entirely on none, some, or all of the column.");
        }
        else {
            enforcedDomainsBuilder.put(predicateColumnHandle, entry.getValue());
        }
    }
    Map<ColumnHandle, Domain> enforcedDomains = enforcedDomainsBuilder.build();
    checkArgument(
            enforcedDomains.size() + unenforcedDomains.size() == predicateDomains.size(),
            "Enforced tuple domain cannot be determined. Connector returned an unenforced TupleDomain that contains columns not in predicate.");
    return TupleDomain.withColumnDomains(enforcedDomains);
}
 
Example 20
Source File: HivePartitionManager.java    From presto with Apache License 2.0 4 votes vote down vote up
public HivePartitionResult getPartitions(SemiTransactionalHiveMetastore metastore, HiveIdentity identity, ConnectorTableHandle tableHandle, Constraint constraint)
{
    HiveTableHandle hiveTableHandle = (HiveTableHandle) tableHandle;
    TupleDomain<ColumnHandle> effectivePredicate = constraint.getSummary()
            .intersect(hiveTableHandle.getEnforcedConstraint());

    SchemaTableName tableName = hiveTableHandle.getSchemaTableName();
    Optional<HiveBucketHandle> hiveBucketHandle = hiveTableHandle.getBucketHandle();
    List<HiveColumnHandle> partitionColumns = hiveTableHandle.getPartitionColumns();

    if (effectivePredicate.isNone()) {
        return new HivePartitionResult(partitionColumns, ImmutableList.of(), none(), none(), none(), hiveBucketHandle, Optional.empty());
    }

    Table table = metastore.getTable(identity, tableName.getSchemaName(), tableName.getTableName())
            .orElseThrow(() -> new TableNotFoundException(tableName));

    Optional<HiveBucketFilter> bucketFilter = getHiveBucketFilter(table, effectivePredicate);
    TupleDomain<HiveColumnHandle> compactEffectivePredicate = effectivePredicate
            .transform(HiveColumnHandle.class::cast)
            .simplify(domainCompactionThreshold);

    if (partitionColumns.isEmpty()) {
        return new HivePartitionResult(
                partitionColumns,
                ImmutableList.of(new HivePartition(tableName)),
                compactEffectivePredicate,
                effectivePredicate,
                TupleDomain.all(),
                hiveBucketHandle,
                bucketFilter);
    }

    List<Type> partitionTypes = partitionColumns.stream()
            .map(HiveColumnHandle::getType)
            .collect(toList());

    Iterable<HivePartition> partitionsIterable;
    Predicate<Map<ColumnHandle, NullableValue>> predicate = constraint.predicate().orElse(value -> true);
    if (hiveTableHandle.getPartitions().isPresent()) {
        partitionsIterable = hiveTableHandle.getPartitions().get().stream()
                .filter(partition -> partitionMatches(partitionColumns, effectivePredicate, predicate, partition))
                .collect(toImmutableList());
    }
    else {
        List<String> partitionNames = getFilteredPartitionNames(metastore, identity, tableName, partitionColumns, effectivePredicate);
        partitionsIterable = () -> partitionNames.stream()
                // Apply extra filters which could not be done by getFilteredPartitionNames
                .map(partitionName -> parseValuesAndFilterPartition(tableName, partitionName, partitionColumns, partitionTypes, effectivePredicate, predicate))
                .filter(Optional::isPresent)
                .map(Optional::get)
                .iterator();
    }

    // All partition key domains will be fully evaluated, so we don't need to include those
    TupleDomain<ColumnHandle> remainingTupleDomain = effectivePredicate.filter((column, domain) -> !partitionColumns.contains(column));
    TupleDomain<ColumnHandle> enforcedTupleDomain = effectivePredicate.filter((column, domain) -> partitionColumns.contains(column));
    return new HivePartitionResult(partitionColumns, partitionsIterable, compactEffectivePredicate, remainingTupleDomain, enforcedTupleDomain, hiveBucketHandle, bucketFilter);
}