org.apache.iceberg.expressions.Expression Java Examples
The following examples show how to use
org.apache.iceberg.expressions.Expression.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.NOT_IN) { Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream().sorted() .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString(); Assert.assertEquals(expectedLiteral, actual); } else { Literal literal = predicate.literal(); String output = transform.toHumanString(literal.value()); Assert.assertEquals(expectedLiteral, output); } }
Example #2
Source File: ProjectionUtil.java From iceberg with Apache License 2.0 | 6 votes |
static <T> UnboundPredicate<T> truncateLong( String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) { long boundary = pred.literal().value(); switch (pred.op()) { case LT: // adjust closed and then transform ltEq return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1L)); case LT_EQ: return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary)); case GT: // adjust closed and then transform gtEq return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1L)); case GT_EQ: return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary)); case EQ: return predicate(pred.op(), name, transform.apply(boundary)); default: return null; } }
Example #3
Source File: ProjectionUtil.java From iceberg with Apache License 2.0 | 6 votes |
static <T> UnboundPredicate<T> truncateIntegerStrict( String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) { int boundary = pred.literal().value(); switch (pred.op()) { case LT: return predicate(Expression.Operation.LT, name, transform.apply(boundary)); case LT_EQ: return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1)); case GT: return predicate(Expression.Operation.GT, name, transform.apply(boundary)); case GT_EQ: return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1)); case NOT_EQ: return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary)); case EQ: // there is no predicate that guarantees equality because adjacent ints transform to the same value return null; default: return null; } }
Example #4
Source File: FindFiles.java From iceberg with Apache License 2.0 | 6 votes |
/** * Filter results to files in any one of the given partitions. * * @param spec a spec for the partitions * @param partitions a list of StructLike that stores a partition tuple * @return this for method chaining */ public Builder inPartitions(PartitionSpec spec, List<StructLike> partitions) { Preconditions.checkArgument(spec.equals(ops.current().spec(spec.specId())), "Partition spec does not belong to table: %s", table); Expression partitionSetFilter = Expressions.alwaysFalse(); for (StructLike partitionData : partitions) { Expression partFilter = Expressions.alwaysTrue(); for (int i = 0; i < spec.fields().size(); i += 1) { PartitionField field = spec.fields().get(i); partFilter = Expressions.and( partFilter, Expressions.equal(field.name(), partitionData.get(i, Object.class))); } partitionSetFilter = Expressions.or(partitionSetFilter, partFilter); } if (partitionFilter != Expressions.alwaysTrue()) { this.partitionFilter = Expressions.or(partitionFilter, partitionSetFilter); } else { this.partitionFilter = partitionSetFilter; } return this; }
Example #5
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testIntegerStrictLowerBound() { Integer value = 100; Schema schema = new Schema(optional(1, "value", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100"); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100"); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100"); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90"); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[90, 100, 100]"); assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE); }
Example #6
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testBinaryStrict() throws Exception { ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8")); Schema schema = new Schema(optional(1, "value", Types.BinaryType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build(); String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8"))); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8")); assertProjectionStrict(spec, notIn("value", value, anotherValue), Expression.Operation.NOT_IN, String.format("[%s, %s]", expectedValue, expectedValue)); assertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE); }
Example #7
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testBucketLongStrict() { Long value = 100L; Schema schema = new Schema(optional(1, "value", Types.LongType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build(); // the bucket number of the value (i.e. 100) is 6 assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE); assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[6, 7, 8]"); assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE); }
Example #8
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testDecimalStrictLowerBound() { Types.DecimalType type = Types.DecimalType.of(9, 2); BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value(); Schema schema = new Schema(optional(1, "value", type)); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00"); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00"); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00"); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90"); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); BigDecimal delta = new BigDecimal(1); assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.NOT_IN, "[99.00, 100.00, 101.00]"); assertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE); }
Example #9
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testBucketLongInclusive() { Long value = 100L; Schema schema = new Schema(optional(1, "value", Types.LongType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build(); // the bucket number of the value (i.e. 100) is 6 assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "6"); assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusive(spec, in("value", value - 1, value, value + 1), Expression.Operation.IN, "[6, 7, 8]"); assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE); }
Example #10
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testDecimalStrictUpperBound() { Types.DecimalType type = Types.DecimalType.of(9, 2); BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value(); Schema schema = new Schema(optional(1, "value", type)); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90"); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00"); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90"); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90"); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); BigDecimal delta = new BigDecimal(1); assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)), Expression.Operation.NOT_IN, "[98.90, 99.90, 100.90]"); assertProjectionStrictValue(spec, in("value", value, value.subtract(delta)), Expression.Operation.FALSE); }
Example #11
Source File: AllManifestsTable.java From iceberg with Apache License 2.0 | 6 votes |
@Override protected CloseableIterable<FileScanTask> planFiles( TableOperations ops, Snapshot snapshot, Expression rowFilter, boolean ignoreResiduals, boolean caseSensitive, boolean colStats) { String schemaString = SchemaParser.toJson(schema()); String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned()); // Data tasks produce the table schema, not the projection schema and projection is done by processing engines. return CloseableIterable.withNoopClose(Iterables.transform(ops.current().snapshots(), snap -> { if (snap.manifestListLocation() != null) { Expression filter = ignoreResiduals ? Expressions.alwaysTrue() : rowFilter; ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(filter); return new ManifestListReadTask(ops.io(), table().spec(), new BaseFileScanTask( DataFiles.fromManifestList(ops.io().newInputFile(snap.manifestListLocation())), schemaString, specString, residuals)); } else { return StaticDataTask.of( ops.io().newInputFile(ops.current().metadataFileLocation()), snap.allManifests(), manifest -> ManifestsTable.manifestFileToRow(table().spec(), manifest)); } })); }
Example #12
Source File: TestResiduals.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testNotIn() { Schema schema = new Schema( Types.NestedField.optional(50, "dateint", Types.IntegerType.get()), Types.NestedField.optional(51, "hour", Types.IntegerType.get()) ); PartitionSpec spec = PartitionSpec.builderFor(schema) .identity("dateint") .build(); ResidualEvaluator resEval = ResidualEvaluator.of(spec, notIn("dateint", 20170815, 20170816, 20170817), true); Expression residual = resEval.residualFor(Row.of(20180815)); Assert.assertEquals("Residual should be alwaysTrue", alwaysTrue(), residual); residual = resEval.residualFor(Row.of(20170815)); Assert.assertEquals("Residual should be alwaysFalse", alwaysFalse(), residual); }
Example #13
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testIntegerInclusiveUpperBound() { Integer value = 99; Schema schema = new Schema(optional(1, "value", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90"); assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90"); assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100"); assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90"); assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90"); assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusive(spec, in("value", value - 1, value, value + 1), Expression.Operation.IN, "[90, 90, 100]"); assertProjectionInclusiveValue(spec, notIn("value", value, value - 1), Expression.Operation.TRUE); }
Example #14
Source File: Bucket.java From iceberg with Apache License 2.0 | 6 votes |
@Override public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) { if (predicate.term() instanceof BoundTransform) { return ProjectionUtil.projectTransformPredicate(this, name, predicate); } if (predicate.isUnaryPredicate()) { return Expressions.predicate(predicate.op(), name); } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) { return Expressions.predicate( predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value())); } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this); } // comparison predicates can't be projected, notEq can't be projected // TODO: small ranges can be projected. // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected. return null; }
Example #15
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testStringInclusive() { String value = "abcdefg"; Schema schema = new Schema(optional(1, "value", Types.StringType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build(); assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde"); assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde"); assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde"); assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde"); assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde"); assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusive(spec, in("value", value, value + "abc"), Expression.Operation.IN, "[abcde, abcde]"); assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE); }
Example #16
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testStringStrict() { String value = "abcdefg"; Schema schema = new Schema(optional(1, "value", Types.StringType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build(); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde"); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde"); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde"); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde"); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); assertProjectionStrict(spec, notIn("value", value, value + "abc"), Expression.Operation.NOT_IN, "[abcde, abcde]"); assertProjectionStrictValue(spec, in("value", value, value + "abc"), Expression.Operation.FALSE); }
Example #17
Source File: TestDatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(predicate.op(), expectedOp); Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.IN) { Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream().sorted() .map(v -> transform.toHumanString((Integer) v)).collect(Collectors.toList()).toString(); Assert.assertEquals(expectedLiteral, actual); } else { Literal literal = predicate.literal(); String output = transform.toHumanString((int) literal.value()); Assert.assertEquals(expectedLiteral, output); } }
Example #18
Source File: TestDatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testMonthStrictUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build(); assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-12"); assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018-01"); assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-12"); assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017-12"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); Integer anotherDate = (Integer) Literal.of("2017-01-01").to(TYPE).value(); assertProjectionStrict(spec, notIn("date", anotherDate, date), Expression.Operation.NOT_IN, "[2017-01, 2017-12]"); assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE); }
Example #19
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testIntegerInclusiveLowerBound() { Integer value = 100; Schema schema = new Schema(optional(1, "value", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90"); assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100"); assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100"); assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100"); assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100"); assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusive(spec, in("value", value - 1, value, value + 1), Expression.Operation.IN, "[90, 100, 100]"); assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE); }
Example #20
Source File: TestTruncatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testIntegerStrictUpperBound() { Integer value = 99; Schema schema = new Schema(optional(1, "value", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build(); assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90"); assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100"); assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90"); assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90"); assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[90, 90, 100]"); assertProjectionStrictValue(spec, in("value", value, value - 1), Expression.Operation.FALSE); }
Example #21
Source File: TestDatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testYearStrictLowerBound() { Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017"); assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017"); assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017"); assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016"); assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017"); assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE); Integer anotherDate = (Integer) Literal.of("2016-12-31").to(TYPE).value(); assertProjectionStrict(spec, notIn("date", date, anotherDate), Expression.Operation.NOT_IN, "[2016, 2017]"); assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE); }
Example #22
Source File: TestDatesProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testYearInclusiveUpperBound() { Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build(); assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017"); assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017"); assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018"); assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017"); assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017"); assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE); Integer anotherDate = (Integer) Literal.of("2016-01-01").to(TYPE).value(); assertProjectionInclusive(spec, in("date", date, anotherDate), Expression.Operation.IN, "[2016, 2017]"); assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE); }
Example #23
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.inclusive(spec).project(filter); UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(predicate.op(), expectedOp); Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op()); Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.IN) { Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream().sorted() .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString(); Assert.assertEquals(expectedLiteral, actual); } else { Literal literal = predicate.literal(); String output = transform.toHumanString(literal.value()); Assert.assertEquals(expectedLiteral, output); } }
Example #24
Source File: DataTableScan.java From iceberg with Apache License 2.0 | 6 votes |
@Override public CloseableIterable<FileScanTask> planFiles(TableOperations ops, Snapshot snapshot, Expression rowFilter, boolean ignoreResiduals, boolean caseSensitive, boolean colStats) { ManifestGroup manifestGroup = new ManifestGroup(ops.io(), snapshot.dataManifests()) .caseSensitive(caseSensitive) .select(colStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS) .filterData(rowFilter) .specsById(ops.current().specsById()) .ignoreDeleted(); if (ignoreResiduals) { manifestGroup = manifestGroup.ignoreResiduals(); } if (PLAN_SCANS_WITH_WORKER_POOL && snapshot.dataManifests().size() > 1) { manifestGroup = manifestGroup.planWith(ThreadPools.getWorkerPool()); } return manifestGroup.planFiles(); }
Example #25
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter, Expression.Operation expectedOp, String expectedLiteral) { Expression projection = Projections.strict(spec).project(filter); UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection); Assert.assertEquals(expectedOp, predicate.op()); Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op()); Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform(); if (predicate.op() == Expression.Operation.NOT_IN) { Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value); String actual = Lists.newArrayList(values).stream().sorted() .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString(); Assert.assertEquals(expectedLiteral, actual); } else { Literal literal = predicate.literal(); String output = transform.toHumanString(literal.value()); Assert.assertEquals(expectedLiteral, output); } }
Example #26
Source File: TestTimestampsProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testMonthStrictUpperBound() { Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build(); assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12"); assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018-01"); assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12"); assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12"); assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12"); assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE); Long anotherDate = (long) Literal.of("2017-11-02T00:00:00.00000").to(TYPE).value(); assertProjectionStrict(spec, notIn("timestamp", anotherDate, date), Expression.Operation.NOT_IN, "[2017-11, 2017-12]"); assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE); }
Example #27
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testBucketIntegerStrict() { Integer value = 100; Schema schema = new Schema(optional(1, "value", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build(); // the bucket number of the value (i.e. 100) is 6 assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6"); assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE); assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE); assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1), Expression.Operation.NOT_IN, "[6, 7, 8]"); assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE); }
Example #28
Source File: TestBucketingProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testBucketStringInclusive() { String value = "abcdefg"; Schema schema = new Schema(optional(1, "value", Types.StringType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build(); // the bucket number of the value (i.e. "abcdefg") is 4 assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4"); assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE); assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE); assertProjectionInclusive(spec, in("value", value, value + "abc"), Expression.Operation.IN, "[4, 9]"); assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE); }
Example #29
Source File: TestTimestampsProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testYearInclusiveUpperBound() { Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build(); assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017"); assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017"); assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018"); assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017"); assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017"); assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); Long anotherDate = (long) Literal.of("2016-12-31T23:59:59.999999").to(TYPE).value(); assertProjectionInclusive(spec, in("timestamp", date, anotherDate), Expression.Operation.IN, "[2016, 2017]"); assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); }
Example #30
Source File: TestTimestampsProjection.java From iceberg with Apache License 2.0 | 6 votes |
@Test public void testHourInclusiveLowerBound() { Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value(); PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build(); assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09"); assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10"); assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10"); assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01-10"); assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE); Long anotherDate = (long) Literal.of("2016-12-02T00:00:00.00000").to(TYPE).value(); assertProjectionInclusive(spec, in("timestamp", date, anotherDate), Expression.Operation.IN, "[2016-12-02-00, 2017-12-01-10]"); assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE); }