Java Code Examples for org.apache.iceberg.expressions.Expression

The following examples show how to use org.apache.iceberg.expressions.Expression. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringStrict() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value, value + "abc"),
      Expression.Operation.NOT_IN, "[abcde, abcde]");
  assertProjectionStrictValue(spec, in("value", value, value + "abc"), Expression.Operation.FALSE);
}
 
Example 2
Source Project: iceberg   Source File: FindFiles.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Filter results to files in any one of the given partitions.
 *
 * @param spec a spec for the partitions
 * @param partitions a list of StructLike that stores a partition tuple
 * @return this for method chaining
 */
public Builder inPartitions(PartitionSpec spec, List<StructLike> partitions) {
  Preconditions.checkArgument(spec.equals(ops.current().spec(spec.specId())),
      "Partition spec does not belong to table: %s", table);

  Expression partitionSetFilter = Expressions.alwaysFalse();
  for (StructLike partitionData : partitions) {
    Expression partFilter = Expressions.alwaysTrue();
    for (int i = 0; i < spec.fields().size(); i += 1) {
      PartitionField field = spec.fields().get(i);
      partFilter = Expressions.and(
          partFilter,
          Expressions.equal(field.name(), partitionData.get(i, Object.class)));
    }
    partitionSetFilter = Expressions.or(partitionSetFilter, partFilter);
  }

  if (partitionFilter != Expressions.alwaysTrue()) {
    this.partitionFilter = Expressions.or(partitionFilter, partitionSetFilter);
  } else {
    this.partitionFilter = partitionSetFilter;
  }

  return this;
}
 
Example 3
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBinaryStrict() throws Exception {
  ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
  Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
  String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue);
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
  assertProjectionStrict(spec, notIn("value", value, anotherValue),
      Expression.Operation.NOT_IN, String.format("[%s, %s]", expectedValue, expectedValue));
  assertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}
 
Example 4
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketLongStrict() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example 5
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerStrictLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 100, 100]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example 6
Source Project: iceberg   Source File: ProjectionUtil.java    License: Apache License 2.0 6 votes vote down vote up
static <T> UnboundPredicate<T> truncateIntegerStrict(
    String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) {
  int boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary));
    case LT_EQ:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1));
    case GT:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary));
    case GT_EQ:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1));
    case NOT_EQ:
      return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
    case EQ:
      // there is no predicate that guarantees equality because adjacent ints transform to the same value
      return null;
    default:
      return null;
  }
}
 
Example 7
Source Project: iceberg   Source File: ProjectionUtil.java    License: Apache License 2.0 6 votes vote down vote up
static <T> UnboundPredicate<T> truncateLongStrict(
    String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
  long boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary));
    case LT_EQ:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1L));
    case GT:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary));
    case GT_EQ:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1L));
    case NOT_EQ:
      return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
    case EQ:
      // there is no predicate that guarantees equality because adjacent longs transform to the same value
      return null;
    default:
      return null;
  }
}
 
Example 8
Source Project: iceberg   Source File: ProjectionUtil.java    License: Apache License 2.0 6 votes vote down vote up
static <T> UnboundPredicate<T> truncateLong(
    String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
  long boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      // adjust closed and then transform ltEq
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1L));
    case LT_EQ:
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary));
    case GT:
      // adjust closed and then transform gtEq
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1L));
    case GT_EQ:
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary));
    case EQ:
      return predicate(pred.op(), name, transform.apply(boundary));
    default:
      return null;
  }
}
 
Example 9
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDecimalStrictLowerBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
          Expression.Operation.NOT_IN, "[99.00, 100.00, 101.00]");
  assertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE);
}
 
Example 10
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketLongInclusive() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "6");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[6, 7, 8]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 
Example 11
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDecimalStrictUpperBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
      Expression.Operation.NOT_IN, "[98.90, 99.90, 100.90]");
  assertProjectionStrictValue(spec, in("value", value, value.subtract(delta)), Expression.Operation.FALSE);
}
 
Example 12
Source Project: iceberg   Source File: AllManifestsTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CloseableIterable<FileScanTask> planFiles(
    TableOperations ops, Snapshot snapshot, Expression rowFilter,
    boolean ignoreResiduals, boolean caseSensitive, boolean colStats) {
  String schemaString = SchemaParser.toJson(schema());
  String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned());

  // Data tasks produce the table schema, not the projection schema and projection is done by processing engines.
  return CloseableIterable.withNoopClose(Iterables.transform(ops.current().snapshots(), snap -> {
    if (snap.manifestListLocation() != null) {
      Expression filter = ignoreResiduals ? Expressions.alwaysTrue() : rowFilter;
      ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(filter);
      return new ManifestListReadTask(ops.io(), table().spec(), new BaseFileScanTask(
          DataFiles.fromManifestList(ops.io().newInputFile(snap.manifestListLocation())),
          schemaString, specString, residuals));
    } else {
      return StaticDataTask.of(
          ops.io().newInputFile(ops.current().metadataFileLocation()),
          snap.allManifests(),
          manifest -> ManifestsTable.manifestFileToRow(table().spec(), manifest));
    }
  }));
}
 
Example 13
Source Project: iceberg   Source File: TestResiduals.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNotIn() {
  Schema schema = new Schema(
      Types.NestedField.optional(50, "dateint", Types.IntegerType.get()),
      Types.NestedField.optional(51, "hour", Types.IntegerType.get())
  );

  PartitionSpec spec = PartitionSpec.builderFor(schema)
      .identity("dateint")
      .build();

  ResidualEvaluator resEval = ResidualEvaluator.of(spec,
      notIn("dateint", 20170815, 20170816, 20170817), true);

  Expression residual = resEval.residualFor(Row.of(20180815));
  Assert.assertEquals("Residual should be alwaysTrue", alwaysTrue(), residual);

  residual = resEval.residualFor(Row.of(20170815));
  Assert.assertEquals("Residual should be alwaysFalse", alwaysFalse(), residual);
}
 
Example 14
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerInclusiveUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 90, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value - 1), Expression.Operation.TRUE);
}
 
Example 15
Source Project: iceberg   Source File: Bucket.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate.isUnaryPredicate()) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) {
    return Expressions.predicate(
        predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }

  // comparison predicates can't be projected, notEq can't be projected
  // TODO: small ranges can be projected.
  // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected.
  return null;
}
 
Example 16
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[abcde, abcde]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 
Example 17
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example 18
Source Project: iceberg   Source File: TestDatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString((Integer) v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString((int) literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example 19
Source Project: iceberg   Source File: TestDatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMonthStrictUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-01, 2017-12]");
  assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE);
}
 
Example 20
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerInclusiveLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 100, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 
Example 21
Source Project: iceberg   Source File: TestTruncatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerStrictUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 90, 100]");
  assertProjectionStrictValue(spec, in("value", value, value - 1), Expression.Operation.FALSE);
}
 
Example 22
Source Project: iceberg   Source File: TestDatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testYearStrictLowerBound() {
  Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2016-12-31").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", date, anotherDate),
      Expression.Operation.NOT_IN, "[2016, 2017]");
  assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE);
}
 
Example 23
Source Project: iceberg   Source File: TestDatesProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testYearInclusiveUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE);

  Integer anotherDate = (Integer) Literal.of("2016-01-01").to(TYPE).value();
  assertProjectionInclusive(spec, in("date", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE);
}
 
Example 24
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example 25
Source Project: iceberg   Source File: DataTableScan.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CloseableIterable<FileScanTask> planFiles(TableOperations ops, Snapshot snapshot,
                                                 Expression rowFilter, boolean ignoreResiduals,
                                                 boolean caseSensitive, boolean colStats) {
  ManifestGroup manifestGroup = new ManifestGroup(ops.io(), snapshot.dataManifests())
      .caseSensitive(caseSensitive)
      .select(colStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS)
      .filterData(rowFilter)
      .specsById(ops.current().specsById())
      .ignoreDeleted();

  if (ignoreResiduals) {
    manifestGroup = manifestGroup.ignoreResiduals();
  }

  if (PLAN_SCANS_WITH_WORKER_POOL && snapshot.dataManifests().size() > 1) {
    manifestGroup = manifestGroup.planWith(ThreadPools.getWorkerPool());
  }

  return manifestGroup.planFiles();
}
 
Example 26
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example 27
Source Project: iceberg   Source File: TestTimestampsProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMonthStrictUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build();

  assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);

  Long anotherDate = (long) Literal.of("2017-11-02T00:00:00.00000").to(TYPE).value();
  assertProjectionStrict(spec, notIn("timestamp", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-11, 2017-12]");
  assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE);
}
 
Example 28
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketIntegerStrict() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example 29
Source Project: iceberg   Source File: TestBucketingProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. "abcdefg") is 4
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[4, 9]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 
Example 30
Source Project: iceberg   Source File: TestTimestampsProjection.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testYearInclusiveUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build();

  assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE);

  Long anotherDate = (long) Literal.of("2016-12-31T23:59:59.999999").to(TYPE).value();
  assertProjectionInclusive(spec, in("timestamp", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE);
}