Java Code Examples for io.prestosql.spi.predicate.Domain

The following examples show how to use io.prestosql.spi.predicate.Domain. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: TestPrometheusSplit.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testPredicatePushDownSetsLowerBoundOnly()
        throws Exception
{
    long predicateLowValue = 1568638171999L - 600000L;
    Range lowRange = Range.greaterThanOrEqual(TimestampType.TIMESTAMP, predicateLowValue);
    ValueSet valueSet = ValueSet.ofRanges(lowRange);
    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of(
            (ColumnHandle) new PrometheusColumnHandle("timestamp", TimestampType.TIMESTAMP, 2), testDomain));
    prometheusTableHandle.setPredicate(Optional.of(testTupleDomain));
    io.airlift.units.Duration maxQueryRangeDuration = new io.airlift.units.Duration(120, TimeUnit.SECONDS);
    io.airlift.units.Duration queryChunkSizeDuration = new io.airlift.units.Duration(30, TimeUnit.SECONDS);
    LocalDateTime now = ofInstant(Instant.ofEpochMilli(1568638171999L), ZoneId.systemDefault());
    PrometheusTimeMachine.useFixedClockAt(now);
    TemporalAmount maxQueryAsTime = java.time.Duration.ofMillis(maxQueryRangeDuration.toMillis());
    List<String> splitTimes = PrometheusSplitManager.generateTimesForSplits(now, maxQueryRangeDuration, queryChunkSizeDuration, prometheusTableHandle);

    String earliestSplit = splitTimes.get(0);
    ZonedDateTime earliestSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(earliestSplit)), ZoneId.systemDefault());
    TemporalAmount queryChunkAsTime = java.time.Duration.ofMillis(queryChunkSizeDuration.toMillis());
    ZonedDateTime startOfQuery = earliestSplitAsTime.minus(queryChunkAsTime);
    assertNotEquals(startOfQuery.toLocalDateTime(), now.minus(maxQueryAsTime).minus(java.time.Duration.ofMillis((splitTimes.size() - 1) * OFFSET_MILLIS)));
    assertEquals(startOfQuery.toInstant().toEpochMilli(), Instant.ofEpochMilli(predicateLowValue).toEpochMilli() - ((splitTimes.size() - 1) * OFFSET_MILLIS));
}
 
Example 2
Source Project: presto   Source File: TestInformationSchemaMetadata.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests information schema predicate pushdown when both schema and table name are specified.
 */
@Test
public void testInformationSchemaPredicatePushdown()
{
    TransactionId transactionId = transactionManager.beginTransaction(false);

    ImmutableMap.Builder<ColumnHandle, Domain> domains = new ImmutableMap.Builder<>();
    domains.put(new InformationSchemaColumnHandle("table_schema"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_schema")));
    domains.put(new InformationSchemaColumnHandle("table_name"), Domain.singleValue(VARCHAR, Slices.utf8Slice("test_view")));
    Constraint constraint = new Constraint(TupleDomain.withColumnDomains(domains.build()));

    ConnectorSession session = createNewSession(transactionId);
    ConnectorMetadata metadata = new InformationSchemaMetadata("test_catalog", this.metadata);
    InformationSchemaTableHandle tableHandle = (InformationSchemaTableHandle)
            metadata.getTableHandle(session, new SchemaTableName("information_schema", "views"));
    tableHandle = metadata.applyFilter(session, tableHandle, constraint)
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);
    assertEquals(tableHandle.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "test_view")));
}
 
Example 3
Source Project: presto   Source File: QueryBuilder.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> toConjuncts(
        JdbcClient client,
        ConnectorSession session,
        Connection connection,
        TupleDomain<ColumnHandle> tupleDomain,
        List<TypeAndValue> accumulator)
{
    if (tupleDomain.isNone()) {
        return ImmutableList.of(ALWAYS_FALSE);
    }
    ImmutableList.Builder<String> builder = ImmutableList.builder();
    for (Map.Entry<ColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
        JdbcColumnHandle column = ((JdbcColumnHandle) entry.getKey());
        Domain domain = pushDownDomain(client, session, connection, column, entry.getValue());
        builder.add(toPredicate(column.getColumnName(), domain, column, accumulator));
    }
    return builder.build();
}
 
Example 4
Source Project: presto   Source File: TestLocalDynamicFilterConsumer.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateSingleColumn()
        throws ExecutionException, InterruptedException
{
    SubPlan subplan = subplan(
            "SELECT count() FROM lineitem, orders WHERE lineitem.orderkey = orders.orderkey " +
                    "AND orders.custkey < 10",
            OPTIMIZED_AND_VALIDATED,
            false);
    JoinNode joinNode = searchJoins(subplan.getChildren().get(0).getFragment()).findOnlyElement();
    LocalDynamicFilterConsumer filter = LocalDynamicFilterConsumer.create(joinNode, ImmutableList.copyOf(subplan.getFragment().getSymbols().values()), 1);
    DynamicFilterId filterId = getOnlyElement(filter.getBuildChannels().keySet());
    Symbol probeSymbol = getOnlyElement(joinNode.getCriteria()).getLeft();

    filter.getTupleDomainConsumer().accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            filterId, Domain.singleValue(BIGINT, 3L))));
    assertEquals(filter.getNodeLocalDynamicFilterForSymbols().get(), ImmutableMap.of(
            probeSymbol, Domain.singleValue(BIGINT, 3L)));
}
 
Example 5
Source Project: presto   Source File: TestShardMetadataRecordCursor.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNoTableFilter()
{
    // Create "orders" table in a different schema
    createTable(tableMetadataBuilder(new SchemaTableName("test", "orders2"))
            .column("orderkey", BIGINT)
            .build());

    // Create another table that should not be selected
    createTable(tableMetadataBuilder(new SchemaTableName("schema1", "foo"))
            .column("orderkey", BIGINT)
            .build());

    TupleDomain<Integer> tupleDomain = TupleDomain.withColumnDomains(
            ImmutableMap.<Integer, Domain>builder()
                    .put(0, Domain.singleValue(createVarcharType(10), utf8Slice("test")))
                    .build());

    MetadataDao metadataDao = dummyHandle.attach(MetadataDao.class);
    Set<Long> actual = ImmutableSet.copyOf(ShardMetadataRecordCursor.getTableIds(dbi, tupleDomain));
    Set<Long> expected = ImmutableSet.of(
            metadataDao.getTableInformation("test", "orders").getTableId(),
            metadataDao.getTableInformation("test", "orders2").getTableId());
    assertEquals(actual, expected);
}
 
Example 6
Source Project: presto   Source File: TestParquetPredicateUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testParquetTupleDomainMap()
{
    MapType mapType = new MapType(
            INTEGER,
            INTEGER,
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"),
            methodHandle(TestParquetPredicateUtils.class, "throwUnsupportedOperationException"));

    HiveColumnHandle columnHandle = createBaseColumn("my_map", 0, HiveType.valueOf("map<int,int>"), mapType, REGULAR, Optional.empty());

    TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(mapType)));

    MessageType fileSchema = new MessageType("hive_schema",
            new GroupType(OPTIONAL, "my_map",
                    new GroupType(REPEATED, "map",
                            new PrimitiveType(REQUIRED, INT32, "key"),
                            new PrimitiveType(OPTIONAL, INT32, "value"))));

    Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
    TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain, fileSchema, true);
    assertTrue(tupleDomain.isAll());
}
 
Example 7
Source Project: presto   Source File: ExpressionConverter.java    License: Apache License 2.0 6 votes vote down vote up
public static Expression toIcebergExpression(TupleDomain<IcebergColumnHandle> tupleDomain)
{
    if (tupleDomain.isAll()) {
        return alwaysTrue();
    }
    if (tupleDomain.getDomains().isEmpty()) {
        return alwaysFalse();
    }
    Map<IcebergColumnHandle, Domain> domainMap = tupleDomain.getDomains().get();
    Expression expression = alwaysTrue();
    for (Map.Entry<IcebergColumnHandle, Domain> entry : domainMap.entrySet()) {
        IcebergColumnHandle columnHandle = entry.getKey();
        Domain domain = entry.getValue();
        expression = and(expression, toIcebergExpression(columnHandle.getName(), columnHandle.getType(), domain));
    }
    return expression;
}
 
Example 8
Source Project: presto   Source File: TestDomainTranslator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testInPredicateWithCasts()
{
    assertPredicateTranslates(
            new InPredicate(
                    C_BIGINT.toSymbolReference(),
                    new InListExpression(ImmutableList.of(cast(toExpression(1L, SMALLINT), BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.singleValue(BIGINT, 1L))));

    assertPredicateTranslates(
            new InPredicate(
                    cast(C_SMALLINT, BIGINT),
                    new InListExpression(ImmutableList.of(toExpression(1L, BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_SMALLINT, Domain.singleValue(SMALLINT, 1L))));

    assertUnsupportedPredicate(new InPredicate(
            cast(C_BIGINT, INTEGER),
            new InListExpression(ImmutableList.of(toExpression(1L, INTEGER)))));
}
 
Example 9
Source Project: presto   Source File: TestLocalDynamicFilterConsumer.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleProbeSymbols()
        throws ExecutionException, InterruptedException
{
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(
            ImmutableMultimap.of(new DynamicFilterId("123"), new Symbol("a1"), new DynamicFilterId("123"), new Symbol("a2")),
            ImmutableMap.of(new DynamicFilterId("123"), 0),
            ImmutableMap.of(new DynamicFilterId("123"), INTEGER),
            1);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<Symbol, Domain>> result = filter.getNodeLocalDynamicFilterForSymbols();
    assertFalse(result.isDone());

    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 7L))));
    assertEquals(result.get(), ImmutableMap.of(
            new Symbol("a1"), Domain.singleValue(INTEGER, 7L),
            new Symbol("a2"), Domain.singleValue(INTEGER, 7L)));
}
 
Example 10
Source Project: presto   Source File: TestLocalDynamicFilterConsumer.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultiplePartitions()
        throws ExecutionException, InterruptedException
{
    LocalDynamicFilterConsumer filter = new LocalDynamicFilterConsumer(
            ImmutableMultimap.of(new DynamicFilterId("123"), new Symbol("a")),
            ImmutableMap.of(new DynamicFilterId("123"), 0),
            ImmutableMap.of(new DynamicFilterId("123"), INTEGER),
            2);
    assertEquals(filter.getBuildChannels(), ImmutableMap.of(new DynamicFilterId("123"), 0));
    Consumer<TupleDomain<DynamicFilterId>> consumer = filter.getTupleDomainConsumer();
    ListenableFuture<Map<Symbol, Domain>> result = filter.getNodeLocalDynamicFilterForSymbols();

    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 10L))));

    assertFalse(result.isDone());
    consumer.accept(TupleDomain.withColumnDomains(ImmutableMap.of(
            new DynamicFilterId("123"), Domain.singleValue(INTEGER, 20L))));

    assertEquals(result.get(), ImmutableMap.of(
            new Symbol("a"), Domain.multipleValues(INTEGER, ImmutableList.of(10L, 20L))));
}
 
Example 11
Source Project: presto   Source File: TupleDomainParquetPredicate.java    License: Apache License 2.0 6 votes vote down vote up
private static <F, T> Domain createDomain(Type type, boolean hasNullValue, ParquetRangeStatistics<F> rangeStatistics, Function<F, T> function)
{
    F min = rangeStatistics.getMin();
    F max = rangeStatistics.getMax();

    if (min != null && max != null) {
        return Domain.create(ValueSet.ofRanges(Range.range(type, function.apply(min), true, function.apply(max), true)), hasNullValue);
    }
    if (max != null) {
        return Domain.create(ValueSet.ofRanges(Range.lessThanOrEqual(type, function.apply(max))), hasNullValue);
    }
    if (min != null) {
        return Domain.create(ValueSet.ofRanges(Range.greaterThanOrEqual(type, function.apply(min))), hasNullValue);
    }
    return Domain.create(ValueSet.all(type), hasNullValue);
}
 
Example 12
Source Project: presto   Source File: PinotQueryBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public static Optional<String> getFilterClause(TupleDomain<ColumnHandle> tupleDomain, Optional<String> timePredicate, List<PinotColumnHandle> columnHandles)
{
    ImmutableList.Builder<String> conjunctsBuilder = ImmutableList.builder();
    timePredicate.ifPresent(predicate -> conjunctsBuilder.add(predicate));
    if (!tupleDomain.equals(TupleDomain.all())) {
        for (PinotColumnHandle columnHandle : columnHandles) {
            Domain domain = tupleDomain.getDomains().get().get(columnHandle);
            if (domain != null) {
                conjunctsBuilder.add(toPredicate(columnHandle.getColumnName(), domain));
            }
        }
    }
    List<String> conjuncts = conjunctsBuilder.build();
    if (!conjuncts.isEmpty()) {
        return Optional.of(Joiner.on(" AND ").join(conjuncts));
    }
    else {
        return Optional.empty();
    }
}
 
Example 13
Source Project: presto   Source File: AtopSplitManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transactionHandle, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy)
{
    AtopTableHandle tableHandle = (AtopTableHandle) table;

    List<ConnectorSplit> splits = new ArrayList<>();
    ZonedDateTime end = ZonedDateTime.now(timeZone);
    for (Node node : nodeManager.getWorkerNodes()) {
        ZonedDateTime start = end.minusDays(maxHistoryDays - 1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        while (start.isBefore(end)) {
            ZonedDateTime splitEnd = start.withHour(23).withMinute(59).withSecond(59).withNano(0);
            Domain splitDomain = Domain.create(ValueSet.ofRanges(Range.range(TIMESTAMP_WITH_TIME_ZONE, 1000 * start.toEpochSecond(), true, 1000 * splitEnd.toEpochSecond(), true)), false);
            if (tableHandle.getStartTimeConstraint().overlaps(splitDomain) && tableHandle.getEndTimeConstraint().overlaps(splitDomain)) {
                splits.add(new AtopSplit(node.getHostAndPort(), start.toEpochSecond(), start.getZone()));
            }
            start = start.plusDays(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
        }
    }

    return new FixedSplitSource(splits);
}
 
Example 14
Source Project: presto   Source File: IcebergPageSourceProvider.java    License: Apache License 2.0 6 votes vote down vote up
private static TupleDomain<ColumnDescriptor> getParquetTupleDomain(Map<List<String>, RichColumnDescriptor> descriptorsByPath, TupleDomain<IcebergColumnHandle> effectivePredicate)
{
    if (effectivePredicate.isNone()) {
        return TupleDomain.none();
    }

    ImmutableMap.Builder<ColumnDescriptor, Domain> predicate = ImmutableMap.builder();
    effectivePredicate.getDomains().get().forEach((columnHandle, domain) -> {
        String baseType = columnHandle.getType().getTypeSignature().getBase();
        // skip looking up predicates for complex types as Parquet only stores stats for primitives
        if (!baseType.equals(StandardTypes.MAP) && !baseType.equals(StandardTypes.ARRAY) && !baseType.equals(StandardTypes.ROW)) {
            RichColumnDescriptor descriptor = descriptorsByPath.get(ImmutableList.of(columnHandle.getName()));
            if (descriptor != null) {
                predicate.put(descriptor, domain);
            }
        }
    });
    return TupleDomain.withColumnDomains(predicate.build());
}
 
Example 15
Source Project: presto   Source File: TestDomainTranslator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFromInPredicateWithCastsAndNulls()
{
    assertPredicateIsAlwaysFalse(new InPredicate(
            C_BIGINT.toSymbolReference(),
            new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT)))));

    assertUnsupportedPredicate(not(new InPredicate(
            cast(C_SMALLINT, BIGINT),
            new InListExpression(ImmutableList.of(toExpression(null, BIGINT))))));

    assertPredicateTranslates(
            new InPredicate(
                    C_BIGINT.toSymbolReference(),
                    new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT), toExpression(1L, BIGINT)))),
            withColumnDomains(ImmutableMap.of(C_BIGINT, Domain.create(ValueSet.ofRanges(Range.equal(BIGINT, 1L)), false))));

    assertPredicateIsAlwaysFalse(not(new InPredicate(
            C_BIGINT.toSymbolReference(),
            new InListExpression(ImmutableList.of(cast(toExpression(null, SMALLINT), BIGINT), toExpression(1L, SMALLINT))))));
}
 
Example 16
Source Project: presto   Source File: TestTupleDomainParquetPredicate.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBigintMatchesWithStatistics()
        throws ParquetCorruptionException
{
    RichColumnDescriptor column = new RichColumnDescriptor(
            new ColumnDescriptor(new String[] {"path"}, INT64, 0, 0),
            new PrimitiveType(OPTIONAL, INT64, "Test column"));
    TupleDomain<ColumnDescriptor> effectivePredicate = TupleDomain.withColumnDomains(ImmutableMap.of(
            column,
            Domain.create(ValueSet.of(BIGINT, 42L, 43L, 44L, 404L), false)));
    TupleDomainParquetPredicate parquetPredicate = new TupleDomainParquetPredicate(effectivePredicate, singletonList(column));

    assertTrue(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(32, 42)), ID, true));
    assertFalse(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(30, 40)), ID, true));
    assertFalse(parquetPredicate.matches(2, ImmutableMap.of(column, longColumnStats(1024, 0x10000 + 42)), ID, true));
}
 
Example 17
Source Project: presto   Source File: IoPlanPrinter.java    License: Apache License 2.0 6 votes vote down vote up
private FormattedDomain parseDomain(Domain domain)
{
    ImmutableSet.Builder<FormattedRange> formattedRanges = ImmutableSet.builder();
    Type type = domain.getType();

    domain.getValues().getValuesProcessor().consume(
            ranges -> formattedRanges.addAll(
                    ranges.getOrderedRanges().stream()
                            .map(range -> new FormattedRange(formatMarker(range.getLow()), formatMarker(range.getHigh())))
                            .collect(toImmutableSet())),
            discreteValues -> formattedRanges.addAll(
                    discreteValues.getValues().stream()
                            .map(value -> valuePrinter.castToVarcharOrFail(type, value))
                            .map(value -> new FormattedMarker(Optional.of(value), EXACTLY))
                            .map(marker -> new FormattedRange(marker, marker))
                            .collect(toImmutableSet())),
            allOrNone -> {
                throw new IllegalStateException("Unreachable AllOrNone consumer");
            });

    return new FormattedDomain(domain.isNullAllowed(), formattedRanges.build());
}
 
Example 18
Source Project: presto   Source File: TestParquetPredicateUtils.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testParquetTupleDomainStructArray()
{
    RowType.Field rowField = new RowType.Field(Optional.of("a"), INTEGER);
    RowType rowType = RowType.from(ImmutableList.of(rowField));

    HiveColumnHandle columnHandle = createBaseColumn("my_array_struct", 0, HiveType.valueOf("array<struct<a:int>>"), rowType, REGULAR, Optional.empty());

    TupleDomain<HiveColumnHandle> domain = withColumnDomains(ImmutableMap.of(columnHandle, Domain.notNull(new ArrayType(rowType))));

    MessageType fileSchema = new MessageType("hive_schema",
            new GroupType(OPTIONAL, "my_array_struct",
                    new GroupType(REPEATED, "bag",
                            new GroupType(OPTIONAL, "array_element", new PrimitiveType(OPTIONAL, INT32, "a")))));

    Map<List<String>, RichColumnDescriptor> descriptorsByPath = getDescriptors(fileSchema, fileSchema);
    TupleDomain<ColumnDescriptor> tupleDomain = getParquetTupleDomain(descriptorsByPath, domain, fileSchema, true);
    assertTrue(tupleDomain.isAll());
}
 
Example 19
@Test
public void consumesDeterministicPredicateIfNewDomainIsNarrower()
{
    ColumnHandle columnHandle = new TpchColumnHandle("nationkey", BIGINT);
    tester().assertThat(removeRedundantTableScanPredicate)
            .on(p -> p.filter(expression("nationkey = BIGINT '44' OR nationkey = BIGINT '45' OR nationkey = BIGINT '47'"),
                    p.tableScan(
                            nationTableHandle,
                            ImmutableList.of(p.symbol("nationkey", BIGINT)),
                            ImmutableMap.of(p.symbol("nationkey", BIGINT), columnHandle),
                            TupleDomain.withColumnDomains(ImmutableMap.of(columnHandle, Domain.multipleValues(BIGINT, ImmutableList.of(44L, 45L, 46L)))))))
            .matches(
                    filter(
                            expression("nationkey IN (BIGINT '44', BIGINT '45')"),
                            constrainedTableScanWithTableLayout(
                                    "nation",
                                    ImmutableMap.of("nationkey", Domain.multipleValues(BIGINT, ImmutableList.of(44L, 45L, 46L))),
                                    ImmutableMap.of("nationkey", "nationkey"))));
}
 
Example 20
Source Project: presto   Source File: TestLocalDynamicFiltersCollector.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCollectorMultipleScansNone()
{
    Symbol symbol1 = new Symbol("symbol1");
    Symbol symbol2 = new Symbol("symbol2");
    Set<Symbol> probeSymbols1 = ImmutableSet.of(symbol1);
    Set<Symbol> probeSymbols2 = ImmutableSet.of(symbol2);

    LocalDynamicFiltersCollector collector = new LocalDynamicFiltersCollector();
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.all());
    assertEquals(collector.getDynamicFilter(probeSymbols2), TupleDomain.all());

    collector.addDynamicFilter(toDomainMap(symbol1, 1L, 2L));
    collector.addDynamicFilter(toDomainMap(symbol2, 2L, 3L));

    collector.addDynamicFilter(ImmutableMap.of(symbol1, Domain.none(BIGINT)));
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.none());
    assertEquals(collector.getDynamicFilter(probeSymbols2), tupleDomain(symbol2, 2L, 3L));
    assertEquals(collector.getDynamicFilter(ImmutableSet.of(symbol1, symbol2)), TupleDomain.none());

    collector.addDynamicFilter(toDomainMap(symbol1, 1L, 2L));
    assertEquals(collector.getDynamicFilter(probeSymbols1), TupleDomain.none());
    assertEquals(collector.getDynamicFilter(probeSymbols2), tupleDomain(symbol2, 2L, 3L));
    assertEquals(collector.getDynamicFilter(ImmutableSet.of(symbol1, symbol2)), TupleDomain.none());
}
 
Example 21
Source Project: presto   Source File: TestDomainTranslator.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testConjunctExpression()
{
    Expression expression = and(
            comparison(GREATER_THAN, C_DOUBLE.toSymbolReference(), doubleLiteral(0)),
            comparison(GREATER_THAN, C_BIGINT.toSymbolReference(), bigintLiteral(0)));
    assertPredicateTranslates(
            expression,
            withColumnDomains(ImmutableMap.of(
                    C_DOUBLE, Domain.create(ValueSet.ofRanges(Range.greaterThan(DOUBLE, .0)), false),
                    C_BIGINT, Domain.create(ValueSet.ofRanges(Range.greaterThan(BIGINT, 0L)), false))));

    assertEquals(
            toPredicate(fromPredicate(expression).getTupleDomain()),
            and(
                    comparison(GREATER_THAN, C_DOUBLE.toSymbolReference(), doubleLiteral(0)),
                    comparison(GREATER_THAN, C_BIGINT.toSymbolReference(), bigintLiteral(0))));
}
 
Example 22
Source Project: presto   Source File: TestPrometheusSplit.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPredicatePushDownSetsUpperBoundOnly()
        throws Exception
{
    long predicateHighValue = 1568638171999L;
    Range highRange = Range.lessThanOrEqual(TimestampType.TIMESTAMP, predicateHighValue);
    ValueSet valueSet = ValueSet.ofRanges(highRange);

    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of(
            (ColumnHandle) new PrometheusColumnHandle("timestamp", TimestampType.TIMESTAMP, 2), testDomain));
    prometheusTableHandle.setPredicate(Optional.of(testTupleDomain));
    io.airlift.units.Duration maxQueryRangeDuration = new io.airlift.units.Duration(120, TimeUnit.SECONDS);
    io.airlift.units.Duration queryChunkSizeDuration = new io.airlift.units.Duration(30, TimeUnit.SECONDS);
    LocalDateTime now = ofInstant(Instant.ofEpochMilli(1568638171999L + 600000L), ZoneId.systemDefault());
    PrometheusTimeMachine.useFixedClockAt(now);

    List<String> splitTimes = PrometheusSplitManager.generateTimesForSplits(now, maxQueryRangeDuration, queryChunkSizeDuration, prometheusTableHandle);

    TemporalAmount expectedMaxQueryAsTime = java.time.Duration.ofMillis(maxQueryRangeDuration.toMillis() +
            ((splitTimes.size() - 1) * OFFSET_MILLIS));
    String lastSplit = splitTimes.get(splitTimes.size() - 1);
    ZonedDateTime lastSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(lastSplit)), ZoneId.systemDefault());
    String earliestSplit = splitTimes.get(0);
    ZonedDateTime earliestSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(earliestSplit)), ZoneId.systemDefault());
    TemporalAmount queryChunkAsTime = java.time.Duration.ofMillis(queryChunkSizeDuration.toMillis());
    java.time.Duration actualMaxDuration = Duration.between(earliestSplitAsTime
            .minus(queryChunkAsTime), lastSplitAsTime);

    assertEquals(lastSplitAsTime.toInstant().toEpochMilli(), 1568638171999L);
    assertEquals(actualMaxDuration, expectedMaxQueryAsTime);
}
 
Example 23
Source Project: presto   Source File: TestPrometheusSplit.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPredicatePushDownSetsUpperAndLowerBound()
        throws Exception
{
    long predicateHighValue = 1568638171999L;
    Range highRange = Range.equal(TimestampType.TIMESTAMP, predicateHighValue);
    long predicateLowValue = 1568638171999L - 600000L;
    Range lowRange = Range.equal(TimestampType.TIMESTAMP, predicateLowValue);
    ValueSet valueSet = ValueSet.ofRanges(lowRange, highRange);

    Domain testDomain = Domain.create(valueSet, false);
    TupleDomain<ColumnHandle> testTupleDomain = TupleDomain.withColumnDomains(ImmutableMap.<ColumnHandle, Domain>of(
            (ColumnHandle) new PrometheusColumnHandle("timestamp", TimestampType.TIMESTAMP, 2), testDomain));
    prometheusTableHandle.setPredicate(Optional.of(testTupleDomain));
    io.airlift.units.Duration maxQueryRangeDuration = new io.airlift.units.Duration(120, TimeUnit.SECONDS);
    io.airlift.units.Duration queryChunkSizeDuration = new io.airlift.units.Duration(30, TimeUnit.SECONDS);
    LocalDateTime now = ofInstant(Instant.ofEpochMilli(1568638171999L + 1200000L), ZoneId.systemDefault());
    PrometheusTimeMachine.useFixedClockAt(now);

    List<String> splitTimes = PrometheusSplitManager.generateTimesForSplits(now, maxQueryRangeDuration, queryChunkSizeDuration, prometheusTableHandle);

    TemporalAmount expectedMaxQueryAsTime = java.time.Duration.ofMillis(new io.airlift.units.Duration(10, TimeUnit.MINUTES).toMillis() +
            ((splitTimes.size() - 1) * OFFSET_MILLIS));
    String lastSplit = splitTimes.get(splitTimes.size() - 1);
    ZonedDateTime lastSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(lastSplit)), ZoneId.systemDefault());
    String earliestSplit = splitTimes.get(0);
    ZonedDateTime earliestSplitAsTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(longFromDecimalSecondString(earliestSplit)), ZoneId.systemDefault());
    TemporalAmount queryChunkAsTime = java.time.Duration.ofMillis(queryChunkSizeDuration.toMillis());
    java.time.Duration actualMaxDuration = Duration.between(earliestSplitAsTime
            .minus(queryChunkAsTime), lastSplitAsTime);

    assertEquals(lastSplitAsTime.toInstant().toEpochMilli(), 1568638171999L);
    assertEquals(actualMaxDuration, expectedMaxQueryAsTime);
}
 
Example 24
Source Project: presto   Source File: TestMongoSession.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBuildQueryIn()
{
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(ImmutableMap.of(
            COL2, Domain.create(ValueSet.ofRanges(equal(createUnboundedVarcharType(), utf8Slice("hello")), equal(createUnboundedVarcharType(), utf8Slice("world"))), false)));

    Document query = MongoSession.buildQuery(tupleDomain);
    Document expected = new Document(COL2.getName(), new Document("$in", ImmutableList.of("hello", "world")));
    assertEquals(query, expected);
}
 
Example 25
Source Project: presto   Source File: TestInformationSchemaMetadata.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testInformationSchemaPredicatePushdownForEmptyNames()
{
    TransactionId transactionId = transactionManager.beginTransaction(false);
    ConnectorSession session = createNewSession(transactionId);
    ConnectorMetadata metadata = new InformationSchemaMetadata("test_catalog", this.metadata);
    InformationSchemaColumnHandle tableSchemaColumn = new InformationSchemaColumnHandle("table_schema");
    InformationSchemaColumnHandle tableNameColumn = new InformationSchemaColumnHandle("table_name");
    ConnectorTableHandle tableHandle = metadata.getTableHandle(session, new SchemaTableName("information_schema", "tables"));

    // Empty schema name
    InformationSchemaTableHandle filtered = metadata.applyFilter(session, tableHandle, new Constraint(TupleDomain.withColumnDomains(
            ImmutableMap.of(tableSchemaColumn, Domain.singleValue(VARCHAR, Slices.utf8Slice(""))))))
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);

    // "" schema name is valid schema name, but is (currently) valid for QualifiedTablePrefix
    assertEquals(filtered.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "")));

    // Empty table name
    filtered = metadata.applyFilter(session, tableHandle, new Constraint(TupleDomain.withColumnDomains(
            ImmutableMap.of(tableNameColumn, Domain.singleValue(VARCHAR, Slices.utf8Slice(""))))))
            .map(ConstraintApplicationResult::getHandle)
            .map(InformationSchemaTableHandle.class::cast)
            .orElseThrow(AssertionError::new);

    // "" table name is valid schema name, but is (currently) valid for QualifiedTablePrefix
    assertEquals(filtered.getPrefixes(), ImmutableSet.of(new QualifiedTablePrefix("test_catalog", "test_schema", "")));
}
 
Example 26
Source Project: presto   Source File: TestTupleDomainParquetPredicate.java    License: Apache License 2.0 5 votes vote down vote up
private TupleDomain<ColumnDescriptor> getEffectivePredicate(RichColumnDescriptor column, VarcharType type, Slice value)
{
    ColumnDescriptor predicateColumn = new ColumnDescriptor(column.getPath(), column.getPrimitiveType().getPrimitiveTypeName(), 0, 0);
    Domain predicateDomain = singleValue(type, value);
    Map<ColumnDescriptor, Domain> predicateColumns = singletonMap(predicateColumn, predicateDomain);
    return withColumnDomains(predicateColumns);
}
 
Example 27
@Test
public void testBuildClusteringPredicate()
{
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(
            ImmutableMap.of(
                    col1, Domain.singleValue(BIGINT, 23L),
                    col2, Domain.singleValue(BIGINT, 34L),
                    col4, Domain.singleValue(BIGINT, 26L)));
    CassandraClusteringPredicatesExtractor predicatesExtractor = new CassandraClusteringPredicatesExtractor(cassandraTable.getClusteringKeyColumns(), tupleDomain, cassandraVersion);
    String predicate = predicatesExtractor.getClusteringKeyPredicates();
    assertEquals(predicate, "\"clusteringKey1\" = 34");
}
 
Example 28
@Test
public void testGetUnenforcedPredicates()
{
    TupleDomain<ColumnHandle> tupleDomain = TupleDomain.withColumnDomains(
            ImmutableMap.of(
                    col2, Domain.singleValue(BIGINT, 34L),
                    col4, Domain.singleValue(BIGINT, 26L)));
    CassandraClusteringPredicatesExtractor predicatesExtractor = new CassandraClusteringPredicatesExtractor(cassandraTable.getClusteringKeyColumns(), tupleDomain, cassandraVersion);
    TupleDomain<ColumnHandle> unenforcedPredicates = TupleDomain.withColumnDomains(ImmutableMap.of(col4, Domain.singleValue(BIGINT, 26L)));
    assertEquals(predicatesExtractor.getUnenforcedConstraints(), unenforcedPredicates);
}
 
Example 29
Source Project: presto   Source File: MongoSession.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
static Document buildQuery(TupleDomain<ColumnHandle> tupleDomain)
{
    Document query = new Document();
    if (tupleDomain.getDomains().isPresent()) {
        for (Map.Entry<ColumnHandle, Domain> entry : tupleDomain.getDomains().get().entrySet()) {
            MongoColumnHandle column = (MongoColumnHandle) entry.getKey();
            Optional<Document> predicate = buildPredicate(column, entry.getValue());
            predicate.ifPresent(query::putAll);
        }
    }

    return query;
}
 
Example 30
Source Project: presto   Source File: MemoryPageSourceProvider.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public ConnectorPageSource createPageSource(
        ConnectorTransactionHandle transaction,
        ConnectorSession session,
        ConnectorSplit split,
        ConnectorTableHandle table,
        List<ColumnHandle> columns,
        TupleDomain<ColumnHandle> dynamicFilter)
{
    MemorySplit memorySplit = (MemorySplit) split;
    long tableId = memorySplit.getTable();
    int partNumber = memorySplit.getPartNumber();
    int totalParts = memorySplit.getTotalPartsPerWorker();
    long expectedRows = memorySplit.getExpectedRows();
    MemoryTableHandle memoryTable = (MemoryTableHandle) table;
    OptionalDouble sampleRatio = memoryTable.getSampleRatio();

    if (dynamicFilter.isNone()) {
        return new FixedPageSource(ImmutableList.of());
    }
    Map<Integer, Domain> domains = dynamicFilter
            .transform(columns::indexOf)
            .getDomains()
            .get();

    List<Integer> columnIndexes = columns.stream()
            .map(MemoryColumnHandle.class::cast)
            .map(MemoryColumnHandle::getColumnIndex).collect(toList());
    List<Page> pages = pagesStore.getPages(
            tableId,
            partNumber,
            totalParts,
            columnIndexes,
            expectedRows,
            memorySplit.getLimit(),
            sampleRatio);
    return new FixedPageSource(pages.stream()
            .map(page -> applyFilter(page, domains))
            .collect(toList()));
}