io.prestosql.spi.predicate.Domain Java Examples

The following examples show how to use io.prestosql.spi.predicate.Domain. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDomainTranslator.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testConjunctExpression()
{
    Expression expression = and(
            comparison(GREATER_THAN, C_DOUBLE.toSymbolReference(), doubleLiteral(0)),
            comparison(GREATER_THAN, C_BIGINT.toSymbolReference(), bigintLiteral(0)));
    assertPredicateTranslates(
            expression,
            withColumnDomains(ImmutableMap.of(
                    C_DOUBLE, Domain.create(ValueSet.ofRanges(Range.greaterThan(DOUBLE, .0)), false),
                    C_BIGINT, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 0L)), false))));

    assertEquals(
            toPredicate(fromPredicate(expression).getTupleDomain()),
            and(
                    comparison(GREATER_THAN, C_DOUBLE.toSymbolReference(), doubleLiteral(0)),
                    comparison(GREATER_THAN, C_BIGINT.toSymbolReference(), bigintLiteral(0))));
}
 
Example #2
Source File: IcebergPageSourceProvider.java    From presto with Apache License 2.0 6 votes vote down vote up
private static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<IcebergColumnHandle> effectivePredicate)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    effectivePredicate.getDomains().get().forEach((columnHandle, domain) -> {
        String baseType = columnHandle.getType().getTypeSignature().getBase();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (!baseType.equals(StandardTypes.MAP) && !baseType.equals(StandardTypes.ARRAY) && !baseType.equals(StandardTypes.ROW)) {
            RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
            if (descriptor != null) {
                predicate.put(descriptor, domain);
            }
        }
    });
    return TupleDomain.withColumnDomains(predicate.build());
}
 
Example #3
Source File: TestDomainTranslator.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testFromInPredicateWithCastsAndNulls()
{
    assertPredicateIsAlwaysFalse(new InPredicate(
            C_BIGINT.toSymbolReference(),
            new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT)))));

    assertUnsupportedPredicate(not(new InPredicate(
            cast(C_SMALLINT, BIGINT),
            new InListExpression(ImmutableList.of(toExpression(null, BIGINT))))));

    assertPredicateTranslates(
            new InPredicate(
                    C_BIGINT.toSymbolReference(),
                    new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT), toExpression(1L, BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L)), false))));

    assertPredicateIsAlwaysFalse(not(new InPredicate(
            C_BIGINT.toSymbolReference(),
            new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT), toExpression(1L, SMALLINT))))));
}
 
Example #4
Source File: TestParquetPredicateUtils.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testParquetTupleDomainMap()
{
    MapType mapType = new MapType(
            INTEGER,
            INTEGER,
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"));

    HiveColumnHandle columnHandle = createBaseColumn("my_map", 0, HiveType.valueOf("map<int,int>"), mapType, REGULAR, Optional.empty());

    TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(mapType)));

    MessageType fileSchema = new MessageType("hive_schema",
            new GroupType(OPTIONAL, "my_map",
                    new GroupType(REPEATED, "map",
                            new PrimitiveType(REQUIRED, INT32, "key"),
                            new PrimitiveType(OPTIONAL, INT32, "value"))));

    Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
    TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain, fileSchema, true);
    assertTrue(tupleDomain.isAll());
}
 
Example #5
Source File: TestShardMetadataRecordCursor.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testNoTableFilter()
{
    // Create "orders" table in a different schema
    createTable(tableMetadataBuilder(new SchemaTableName("test", "orders2"))
            .column("orderkey", BIGINT)
            .build());

    // Create another table that should not be selected
    createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo"))
            .column("orderkey", BIGINT)
            .build());

    TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains(
            ImmutableMap.<Integer, Domain>builder()
                    .put(0, Domain.singleValue(createVarcharType(10), utf8Slice("test")))
                    .build());

    MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class);
    Set<Long> actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain));
    Set<Long> expected = ImmutableSet.of(
            metadataDao.getTableInformation("test", "orders").getTableId(),
            metadataDao.getTableInformation("test", "orders2").getTableId());
    assertEquals(actual, expected);
}
 
Example #6
Source File: TestLocalDynamicFilterConsumer.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateSingleColumn()
        throws ExecutionException, InterruptedException
{
    SubPlan subplan = subplan(
            "SELECT count() FROM lineitem, orders WHERE lineitem.orderkey = orders.orderkey " +
                    "AND orders.custkey < 10",
            OPTIMIZED_AND_VALIDATED,
            false);
    JoinNode joinNode = searchJoins(subplan.getChildren().get(0).getFragment()).findOnlyElement();
    LocalDynamicFilterConsumer filter = LocalDynamicFilterConsumer.create(joinNode, ImmutableList.copyOf(subplan.getFragment().getSymbols().values()), 1);
    DynamicFilterId filterId = getOnlyElement(filter.getBuildChannels().keySet());
    Symbol probeSymbol = getOnlyElement(joinNode.getCriteria()).getLeft();

    filter.getTupleDomainConsumer().accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            filterId, Domain.singleValue(BIGINT, 3L))));
    assertEquals(filter.getNodeLocalDynamicFilterForSymbols().get(), ImmutableMap.of(
            probeSymbol, Domain.singleValue(BIGINT, 3L)));
}
 
Example #7
Source File: ExpressionConverter.java    From presto with Apache License 2.0 6 votes vote down vote up
public static Expression toIcebergExpression(TupleDomain<IcebergColumnHandle> tupleDomain)
{
    if (tupleDomain.isAll()) {
        return alwaysTrue();
    }
    if (tupleDomain.getDomains().isEmpty()) {
        return alwaysFalse();
    }
    Map<IcebergColumnHandle, Domain> domainMap = tupleDomain.getDomains().get();
    Expression expression = alwaysTrue();
    for (Map.Entry<IcebergColumnHandle, Domain> entry : domainMap.entrySet()) {
        IcebergColumnHandle columnHandle = entry.getKey();
        Domain domain = entry.getValue();
        expression = and(expression, toIcebergExpression(columnHandle.getName(), columnHandle.getType(), domain));
    }
    return expression;
}
 
Example #8
Source File: QueryBuilder.java    From presto with Apache License 2.0 6 votes vote down vote up
private List<String> toConjuncts(
        JdbcClient client,
        ConnectorSession session,
        Connection connection,
        TupleDomain<ColumnHandle> tupleDomain,
        List<TypeAndValue> accumulator)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of(ALWAYS_FALSE);
    }
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
        JdbcColumnHandle column = ((JdbcColumnHandle) entry.getKey());
        Domain domain = pushDownDomain(client, session, connection, column, entry.getValue());
        builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
    }
    return builder.build();
}
 
Example #9
Source File: TestTupleDomainParquetPredicate.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testBigintMatchesWithStatistics()
        throws ParquetCorruptionException
{
    RichColumnDescriptor column = new RichColumnDescriptor(
            new ColumnDescriptor(new String[] {"path"}, INT64, 0, 0),
            new PrimitiveType(OPTIONAL, INT64, "Test column"));
    TupleDomain<ColumnDescriptor> effectivePredicate = TupleDomain.withColumnDomains(ImmutableMap.of(
            column,
            Domain.create(ValueSet.of(BIGINT, 42L, 43L, 44L, 404L), false)));
    TupleDomainParquetPredicate parquetPredicate = new TupleDomainParquetPredicate(effectivePredicate, singletonList(column));

    assertTrue(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(32, 42)), ID, true));
    assertFalse(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(30, 40)), ID, true));
    assertFalse(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(1024, 0x10000 + 42)), ID, true));
}
 
Example #10
Source File: TestDomainTranslator.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testInPredicateWithCasts()
{
    assertPredicateTranslates(
            new InPredicate(
                    C_BIGINT.toSymbolReference(),
                    new InListExpression(ImmutableList.of(cast(toExpression(1L, SMALLINT), BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.singleValue(BIGINT, 1L))));

    assertPredicateTranslates(
            new InPredicate(
                    cast(C_SMALLINT, BIGINT),
                    new InListExpression(ImmutableList.of(toExpression(1L, BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_SMALLINT, Domain.singleValue(SMALLINT, 1L))));

    assertUnsupportedPredicate(new InPredicate(
            cast(C_BIGINT, INTEGER),
            new InListExpression(ImmutableList.of(toExpression(1L, INTEGER)))));
}
 
Example #11
Source File: AtopSplitManager.java    From presto with Apache License 2.0 6 votes vote down vote up
@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy)
{
    AtopTableHandle tableHandle = (AtopTableHandle) table;

    List<ConnectorSplit> splits = new ArrayList<>();
    ZonedDateTime end = ZonedDateTime.now(timeZone);
    for (Node node : nodeManager.getWorkerNodes()) {
        ZonedDateTime start = end.minusDays(maxHistoryDays - 1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        while (start.isBefore(end)) {
            ZonedDateTime splitEnd = start.withHour(23).withMinute(59).withSecond(59).withNano(0);
            Domain splitDomain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP_WITH_TIME_ZONE, 1000 * start.toEpochSecond(), true, 1000 * splitEnd.toEpochSecond(), true)), false);
            if (tableHandle.getStartTimeConstraint().overlaps(splitDomain) && tableHandle.getEndTimeConstraint().overlaps(splitDomain)) {
                splits.add(new AtopSplit(node.getHostAndPort(), start.toEpochSecond(), start.getZone()));
            }
            start = start.plusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        }
    }

    return new FixedSplitSource(splits);
}
 
Example #12
Source File: IoPlanPrinter.java    From presto with Apache License 2.0 6 votes vote down vote up
private FormattedDomain parseDomain(Domain domain)
{
    ImmutableSet.Builder<FormattedRange> formattedRanges = ImmutableSet.builder();
    Type type = domain.getType();

    domain.getValues().getValuesProcessor().consume(
            ranges -> formattedRanges.addAll(
                    ranges.getOrderedRanges().stream()
                            .map(range -> new FormattedRange(formatMarker(range.getLow()), formatMarker(range.getHigh())))
                            .collect(toImmutableSet())),
            discreteValues -> formattedRanges.addAll(
                    discreteValues.getValues().stream()
                            .map(value -> valuePrinter.castToVarcharOrFail(type, value))
                            .map(value -> new FormattedMarker(Optional.of(value), EXACTLY))
                            .map(marker -> new FormattedRange(marker, marker))
                            .collect(toImmutableSet())),
            allOrNone -> {
                throw new IllegalStateException("Unreachable AllOrNone consumer");
            });

    return new FormattedDomain(domain.isNullAllowed(), formattedRanges.build());
}
 
Example #13
Source File: TestParquetPredicateUtils.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testParquetTupleDomainStructArray()
{
    RowType.Field rowField = new RowType.Field(Optional.of("a"), INTEGER);
    RowType rowType = RowType.from(ImmutableList.of(rowField));

    HiveColumnHandle columnHandle = createBaseColumn("my_array_struct", 0, HiveType.valueOf("array<struct<a:int>>"), rowType, REGULAR, Optional.empty());

    TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(rowType))));

    MessageType fileSchema = new MessageType("hive_schema",
            new GroupType(OPTIONAL, "my_array_struct",
                    new GroupType(REPEATED, "bag",
                            new GroupType(OPTIONAL, "array_element", new PrimitiveType(OPTIONAL, INT32, "a")))));

    Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
    TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain, fileSchema, true);
    assertTrue(tupleDomain.isAll());
}
 
Example #14
Source File: TestRemoveRedundantTableScanPredicate.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void consumesDeterministicPredicateIfNewDomainIsNarrower()
{
    ColumnHandle columnHandle = new TpchColumnHandle("nationkey", BIGINT);
    tester().assertThat(removeRedundantTableScanPredicate)
            .on(p -> p.filter(expression("nationkey = BIGINT '44' OR nationkey = BIGINT '45' OR nationkey = BIGINT '47'"),
                    p.tableScan(
                            nationTableHandle,
                            ImmutableList.of(p.symbol("nationkey", BIGINT)),
                            ImmutableMap.of(p.symbol("nationkey", BIGINT), columnHandle),
                            TupleDomain.withColumnDomains(ImmutableMap.of(columnHandle, Domain.multipleValues(BIGINT, ImmutableList.of(44L, 45L, 46L)))))))
            .matches(
                    filter(
                            expression("nationkey IN (BIGINT '44', BIGINT '45')"),
                            constrainedTableScanWithTableLayout(
                                    "nation",
                                    ImmutableMap.of("nationkey", Domain.multipleValues(BIGINT, ImmutableList.of(44L, 45L, 46L))),
                                    ImmutableMap.of("nationkey", "nationkey"))));
}
 
Example #15
Source File: TestLocalDynamicFiltersCollector.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testCollectorMultipleScansNone()
{
    Symbol symbol1 = new Symbol("symbol1");
    Symbol symbol2 = new Symbol("symbol2");
    Set<Symbol> probeSymbols1 = ImmutableSet.of(symbol1);
    Set<Symbol> probeSymbols2 = ImmutableSet.of(symbol2);

    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector();
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.all());
    assertEquals(collector.getDynamicFilter(probeSymbols2), TupleDomain.all());

    collector.addDynamicFilter(toDomainMap(symbol1, 1L, 2L));
    collector.addDynamicFilter(toDomainMap(symbol2, 2L, 3L));

    collector.addDynamicFilter(ImmutableMap.of(symbol1, Domain.none(BIGINT)));
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.none());
    assertEquals(collector.getDynamicFilter(probeSymbols2), tupleDomain(symbol2, 2L, 3L));
    assertEquals(collector.getDynamicFilter(ImmutableSet.of(symbol1, symbol2)), TupleDomain.none());

    collector.addDynamicFilter(toDomainMap(symbol1, 1L, 2L));
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.none());
    assertEquals(collector.getDynamicFilter(probeSymbols2), tupleDomain(symbol2, 2L, 3L));
    assertEquals(collector.getDynamicFilter(ImmutableSet.of(symbol1, symbol2)), TupleDomain.none());
}
 
Example #16
Source File: TestLocalDynamicFilterConsumer.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleProbeSymbols()
        throws ExecutionException, InterruptedException
{
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(
            ImmutableMultimap.of(new DynamicFilterId("123"), new Symbol("a1"), new DynamicFilterId("123"), new Symbol("a2")),
            ImmutableMap.of(new DynamicFilterId("123"), 0),
            ImmutableMap.of(new DynamicFilterId("123"), INTEGER),
            1);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<Symbol, Domain>> result = filter.getNodeLocalDynamicFilterForSymbols();
    assertFalse(result.isDone());

    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 7L))));
    assertEquals(result.get(), ImmutableMap.of(
            new Symbol("a1"), Domain.singleValue(INTEGER, 7L),
            new Symbol("a2"), Domain.singleValue(INTEGER, 7L)));
}
 
Example #17
Source File: TestLocalDynamicFilterConsumer.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiplePartitions()
        throws ExecutionException, InterruptedException
{
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(
            ImmutableMultimap.of(new DynamicFilterId("123"), new Symbol("a")),
            ImmutableMap.of(new DynamicFilterId("123"), 0),
            ImmutableMap.of(new DynamicFilterId("123"), INTEGER),
            2);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<Symbol, Domain>> result = filter.getNodeLocalDynamicFilterForSymbols();

    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 10L))));

    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 20L))));

    assertEquals(result.get(), ImmutableMap.of(
            new Symbol("a"), Domain.multipleValues(INTEGER, ImmutableList.of(10L, 20L))));
}
 
Example #18
Source File: TestPrometheusSplit.java    From presto with Apache License 2.0 6 votes vote down vote up
@Test
public void testPredicatePushDownSetsLowerBoundOnly()
        throws Exception
{
    long predicateLowValue = 1568638171999L - 600000L;
    Range lowRange = Range.greaterThanOrEqual(TimestampType.TIMESTAMP, predicateLowValue);
    ValueSet valueSet = ValueSet.ofRanges(lowRange);
    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of(
            (ColumnHandle) new PrometheusColumnHandle("timestamp", TimestampType.TIMESTAMP, 2), testDomain));
    prometheusTableHandle.setPredicate(Optional.of(testTupleDomain));
    io.airlift.units.Duration maxQueryRangeDuration = new io.airlift.units.Duration(120, TimeUnit.SECONDS);
    io.airlift.units.Duration queryChunkSizeDuration = new io.airlift.units.Duration(30, TimeUnit.SECONDS);
    LocalDateTime now = ofInstant(Instant.ofEpochMilli(1568638171999L), ZoneId.systemDefault());
    PrometheusTimeMachine.useFixedClockAt(now);
    TemporalAmount maxQueryAsTime = java.time.Duration.ofMillis(maxQueryRangeDuration.toMillis());
    List<String> splitTimes = PrometheusSplitManager.generateTimesForSplits(now, maxQueryRangeDuration, queryChunkSizeDuration, prometheusTableHandle);

    String earliestSplit = splitTimes.get(0);
    ZonedDateTime earliestSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(earliestSplit)), ZoneId.systemDefault());
    TemporalAmount queryChunkAsTime = java.time.Duration.ofMillis(queryChunkSizeDuration.toMillis());
    ZonedDateTime startOfQuery = earliestSplitAsTime.minus(queryChunkAsTime);
    assertNotEquals(startOfQuery.toLocalDateTime(), now.minus(maxQueryAsTime).minus(java.time.Duration.ofMillis((splitTimes.size() - 1) * OFFSET_MILLIS)));
    assertEquals(startOfQuery.toInstant().toEpochMilli(), Instant.ofEpochMilli(predicateLowValue).toEpochMilli() - ((splitTimes.size() - 1) * OFFSET_MILLIS));
}
 
Example #19
Source File: PinotQueryBuilder.java    From presto with Apache License 2.0 6 votes vote down vote up
public static Optional<String> getFilterClause(TupleDomain<ColumnHandle> tupleDomain, Optional<String> timePredicate, List<PinotColumnHandle> columnHandles)
{
    ImmutableList.Builder<String> conjunctsBuilder = ImmutableList.builder();
    timePredicate.ifPresent(predicate -> conjunctsBuilder.add(predicate));
    if (!tupleDomain.equals(TupleDomain.all())) {
        for (PinotColumnHandle columnHandle : columnHandles) {
            Domain domain = tupleDomain.getDomains().get().get(columnHandle);
            if (domain != null) {
                conjunctsBuilder.add(toPredicate(columnHandle.getColumnName(), domain));
            }
        }
    }
    List<String> conjuncts = conjunctsBuilder.build();
    if (!conjuncts.isEmpty()) {
        return Optional.of(Joiner.on(" AND ").join(conjuncts));
    }
    else {
        return Optional.empty();
    }
}
 
Example #20
Source File: TestInformationSchemaMetadata.java    From presto with Apache License 2.0 6 votes vote down vote up
/**
 * Tests information schema predicate pushdown when both schema and table name are specified.
 */
@Test
public void testInformationSchemaPredicatePushdown()
{
    TransactionId transactionId = transactionManager.beginTransaction(false);

    ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>();
    domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema")));
    domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view")));
    Constraint constraint = new Constraint(TupleDomain.withColumnDomains(domains.build()));

    ConnectorSession session = createNewSession(transactionId);
    ConnectorMetadata metadata = new InformationSchemaMetadata("test_catalog", this.metadata);
    InformationSchemaTableHandle tableHandle = (InformationSchemaTableHandle)
            metadata.getTableHandle(session, new SchemaTableName("information_schema", "views"));
    tableHandle = metadata.applyFilter(session, tableHandle, constraint)
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);
    assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view")));
}
 
Example #21
Source File: TupleDomainParquetPredicate.java    From presto with Apache License 2.0 6 votes vote down vote up
private static <F, T> Domain createDomain(Type type, boolean hasNullValue, ParquetRangeStatistics<F> rangeStatistics, Function<F, T> function)
{
    F min = rangeStatistics.getMin();
    F max = rangeStatistics.getMax();

    if (min != null && max != null) {
        return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue);
    }
    if (max != null) {
        return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue);
    }
    if (min != null) {
        return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue);
    }
    return Domain.create(ValueSet.all(type), hasNullValue);
}
 
Example #22
Source File: DomainTranslator.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
protected ExtractionResult visitIsNullPredicate(IsNullPredicate node, Boolean complement)
{
    if (!(node.getValue() instanceof SymbolReference)) {
        return super.visitIsNullPredicate(node, complement);
    }

    Symbol symbol = Symbol.from(node.getValue());
    Type columnType = checkedTypeLookup(symbol);
    Domain domain = complementIfNecessary(Domain.onlyNull(columnType), complement);
    return new ExtractionResult(
            TupleDomain.withColumnDomains(ImmutableMap.of(symbol, domain)),
            TRUE_LITERAL);
}
 
Example #23
Source File: TestDynamicFilterSourceOperator.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testCollectOnlyLastColumn()
{
    OperatorFactory operatorFactory = createOperatorFactory(channel(1, DOUBLE));
    verifyPassthrough(createOperator(operatorFactory),
            ImmutableList.of(BOOLEAN, DOUBLE),
            new Page(createBooleansBlock(true, 2), createDoublesBlock(1.5, 3.0)),
            new Page(createBooleansBlock(false, 1), createDoublesBlock(4.5)));
    operatorFactory.noMoreOperators();

    assertEquals(partitions.build(), ImmutableList.of(
            TupleDomain.withColumnDomains(ImmutableMap.of(
                    new DynamicFilterId("1"), Domain.multipleValues(DOUBLE, ImmutableList.of(1.5, 3.0, 4.5))))));
}
 
Example #24
Source File: DomainTranslator.java    From presto with Apache License 2.0 5 votes vote down vote up
private static Domain extractEquatableDomain(ComparisonExpression.Operator comparisonOperator, Type type, Object value, boolean complement)
{
    checkArgument(value != null);
    switch (comparisonOperator) {
        case EQUAL:
            return Domain.create(complementIfNecessary(ValueSet.of(type, value), complement), false);
        case NOT_EQUAL:
            return Domain.create(complementIfNecessary(ValueSet.of(type, value).complement(), complement), false);
        case IS_DISTINCT_FROM:
            // Need to potential complement the whole domain for IS_DISTINCT_FROM since it is null-aware
            return complementIfNecessary(Domain.create(ValueSet.of(type, value).complement(), true), complement);
        default:
            throw new AssertionError("Unhandled operator: " + comparisonOperator);
    }
}
 
Example #25
Source File: TestDomainConverter.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testReal()
{
    assertTupleDomainUnchanged(
            TupleDomain.withColumnDomains(
                    ImmutableMap.of(ICEBERG_COLUMN_PROVIDER.apply(REAL), Domain.singleValue(REAL, 1L))));

    assertTupleDomainUnchanged(
            TupleDomain.withColumnDomains(
                    ImmutableMap.of(ICEBERG_COLUMN_PROVIDER.apply(REAL), Domain.multipleValues(REAL, ImmutableList.of(1L, 2L, 3L)))));
}
 
Example #26
Source File: TupleDomainParquetPredicate.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public boolean matches(long numberOfRows, Map<ColumnDescriptor, Statistics<?>> statistics, ParquetDataSourceId id, boolean failOnCorruptedParquetStatistics)
        throws ParquetCorruptionException
{
    if (numberOfRows == 0) {
        return false;
    }
    if (effectivePredicate.isNone()) {
        return false;
    }
    Map<ColumnDescriptor, Domain> effectivePredicateDomains = effectivePredicate.getDomains()
            .orElseThrow(() -> new IllegalStateException("Effective predicate other than none should have domains"));

    for (RichColumnDescriptor column : columns) {
        Domain effectivePredicateDomain = effectivePredicateDomains.get(column);
        if (effectivePredicateDomain == null) {
            continue;
        }

        Statistics<?> columnStatistics = statistics.get(column);
        if (columnStatistics == null || columnStatistics.isEmpty()) {
            // no stats for column
            continue;
        }

        Domain domain = getDomain(effectivePredicateDomain.getType(), numberOfRows, columnStatistics, id, column.toString(), failOnCorruptedParquetStatistics);
        if (!effectivePredicateDomain.overlaps(domain)) {
            return false;
        }
    }
    return true;
}
 
Example #27
Source File: ColumnJdbcTable.java    From presto with Apache License 2.0 5 votes vote down vote up
private static Collector<String, ?, Domain> toVarcharDomain()
{
    return Collectors.collectingAndThen(toImmutableSet(), set -> {
        if (set.isEmpty()) {
            return Domain.none(createUnboundedVarcharType());
        }
        return Domain.multipleValues(createUnboundedVarcharType(), set.stream()
                .map(Slices::utf8Slice)
                .collect(toImmutableList()));
    });
}
 
Example #28
Source File: DynamicFilterService.java    From presto with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public void collectDynamicFilters()
{
    for (Map.Entry<QueryId, Supplier<List<StageDynamicFilters>>> entry : getQueries().entrySet()) {
        QueryId queryId = entry.getKey();
        ImmutableMap.Builder<SourceDescriptor, Domain> newDynamicFiltersBuilder = ImmutableMap.builder();
        for (StageDynamicFilters stageDynamicFilters : entry.getValue().get()) {
            StageState stageState = stageDynamicFilters.getStageState();
            // wait until stage has finished scheduling tasks
            if (stageState.canScheduleMoreTasks()) {
                continue;
            }
            stageDynamicFilters.getTaskDynamicFilters().stream()
                    .flatMap(taskDomains -> taskDomains.entrySet().stream())
                    .filter(domain -> !dynamicFilterSummaries.containsKey(SourceDescriptor.of(queryId, domain.getKey())))
                    .collect(groupingBy(Map.Entry::getKey, mapping(Map.Entry::getValue, toImmutableList())))
                    .entrySet().stream()
                    // check if all tasks of a dynamic filter source have reported dynamic filter summary
                    .filter(stageDomains -> stageDomains.getValue().size() == stageDynamicFilters.getNumberOfTasks())
                    .forEach(stageDomains -> newDynamicFiltersBuilder.put(
                            SourceDescriptor.of(queryId, stageDomains.getKey()),
                            Domain.union(stageDomains.getValue())));
        }

        Map<SourceDescriptor, Domain> newDynamicFilters = newDynamicFiltersBuilder.build();
        if (!newDynamicFilters.isEmpty()) {
            addDynamicFilters(queryId, newDynamicFilters);
        }
    }
}
 
Example #29
Source File: TestLogicalPlanner.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testLikePredicate()
{
    assertPlan("SELECT type FROM part WHERE type LIKE 'LARGE PLATED %'",
            anyTree(
                    tableScan(
                            tableHandle -> {
                                Map<ColumnHandle, Domain> domains = ((TpchTableHandle) tableHandle).getConstraint().getDomains()
                                        .orElseThrow(() -> new AssertionError("Unexpected none TupleDomain"));

                                Domain domain = domains.entrySet().stream()
                                        .filter(entry -> ((TpchColumnHandle) entry.getKey()).getColumnName().equals("type"))
                                        .map(Entry::getValue)
                                        .collect(toOptional())
                                        .orElseThrow(() -> new AssertionError("No domain for 'type'"));

                                assertEquals(domain, Domain.multipleValues(
                                        createVarcharType(25),
                                        ImmutableList.of("LARGE PLATED BRASS", "LARGE PLATED COPPER", "LARGE PLATED NICKEL", "LARGE PLATED STEEL", "LARGE PLATED TIN").stream()
                                                .map(Slices::utf8Slice)
                                                .collect(toImmutableList())));
                                return true;
                            },
                            TupleDomain.withColumnDomains(ImmutableMap.of(
                                    tableHandle -> ((TpchColumnHandle) tableHandle).getColumnName().equals("type"),
                                    Domain.create(
                                            ValueSet.ofRanges(Range.range(createVarcharType(25), utf8Slice("LARGE PLATED "), true, utf8Slice("LARGE PLATED!"), false)),
                                            false))),
                            ImmutableMap.of())));
}
 
Example #30
Source File: ShardPredicate.java    From presto with Apache License 2.0 5 votes vote down vote up
private static String createShardPredicate(ImmutableList.Builder<JDBCType> types, ImmutableList.Builder<Object> values, Domain domain, JDBCType jdbcType)
{
    List<Range> ranges = domain.getValues().getRanges().getOrderedRanges();

    // only apply predicates if all ranges are single values
    if (ranges.isEmpty() || !ranges.stream().allMatch(Range::isSingleValue)) {
        return "true";
    }

    ImmutableList.Builder<Object> valuesBuilder = ImmutableList.builder();
    ImmutableList.Builder<JDBCType> typesBuilder = ImmutableList.builder();

    StringJoiner rangePredicate = new StringJoiner(" OR ");
    for (Range range : ranges) {
        Slice uuidText = (Slice) range.getSingleValue();
        try {
            Slice uuidBytes = uuidStringToBytes(uuidText);
            typesBuilder.add(jdbcType);
            valuesBuilder.add(uuidBytes);
        }
        catch (IllegalArgumentException e) {
            return "true";
        }
        rangePredicate.add("shard_uuid = ?");
    }

    types.addAll(typesBuilder.build());
    values.addAll(valuesBuilder.build());
    return rangePredicate.toString();
}