org.apache.iceberg.expressions.Expression Java Examples

The following examples show how to use org.apache.iceberg.expressions.Expression. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example #2
Source File: ProjectionUtil.java    From iceberg with Apache License 2.0 6 votes vote down vote up
static <T> UnboundPredicate<T> truncateLong(
    String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
  long boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      // adjust closed and then transform ltEq
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1L));
    case LT_EQ:
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary));
    case GT:
      // adjust closed and then transform gtEq
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1L));
    case GT_EQ:
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary));
    case EQ:
      return predicate(pred.op(), name, transform.apply(boundary));
    default:
      return null;
  }
}
 
Example #3
Source File: ProjectionUtil.java    From iceberg with Apache License 2.0 6 votes vote down vote up
static <T> UnboundPredicate<T> truncateIntegerStrict(
    String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) {
  int boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary));
    case LT_EQ:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1));
    case GT:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary));
    case GT_EQ:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1));
    case NOT_EQ:
      return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
    case EQ:
      // there is no predicate that guarantees equality because adjacent ints transform to the same value
      return null;
    default:
      return null;
  }
}
 
Example #4
Source File: FindFiles.java    From iceberg with Apache License 2.0 6 votes vote down vote up
/**
 * Filter results to files in any one of the given partitions.
 *
 * @param spec a spec for the partitions
 * @param partitions a list of StructLike that stores a partition tuple
 * @return this for method chaining
 */
public Builder inPartitions(PartitionSpec spec, List<StructLike> partitions) {
  Preconditions.checkArgument(spec.equals(ops.current().spec(spec.specId())),
      "Partition spec does not belong to table: %s", table);

  Expression partitionSetFilter = Expressions.alwaysFalse();
  for (StructLike partitionData : partitions) {
    Expression partFilter = Expressions.alwaysTrue();
    for (int i = 0; i < spec.fields().size(); i += 1) {
      PartitionField field = spec.fields().get(i);
      partFilter = Expressions.and(
          partFilter,
          Expressions.equal(field.name(), partitionData.get(i, Object.class)));
    }
    partitionSetFilter = Expressions.or(partitionSetFilter, partFilter);
  }

  if (partitionFilter != Expressions.alwaysTrue()) {
    this.partitionFilter = Expressions.or(partitionFilter, partitionSetFilter);
  } else {
    this.partitionFilter = partitionSetFilter;
  }

  return this;
}
 
Example #5
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerStrictLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 100, 100]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example #6
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testBinaryStrict() throws Exception {
  ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
  Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
  String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue);
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
  assertProjectionStrict(spec, notIn("value", value, anotherValue),
      Expression.Operation.NOT_IN, String.format("[%s, %s]", expectedValue, expectedValue));
  assertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}
 
Example #7
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketLongStrict() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example #8
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testDecimalStrictLowerBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
          Expression.Operation.NOT_IN, "[99.00, 100.00, 101.00]");
  assertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE);
}
 
Example #9
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketLongInclusive() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "6");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[6, 7, 8]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 
Example #10
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testDecimalStrictUpperBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
      Expression.Operation.NOT_IN, "[98.90, 99.90, 100.90]");
  assertProjectionStrictValue(spec, in("value", value, value.subtract(delta)), Expression.Operation.FALSE);
}
 
Example #11
Source File: AllManifestsTable.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
protected CloseableIterable<FileScanTask> planFiles(
    TableOperations ops, Snapshot snapshot, Expression rowFilter,
    boolean ignoreResiduals, boolean caseSensitive, boolean colStats) {
  String schemaString = SchemaParser.toJson(schema());
  String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned());

  // Data tasks produce the table schema, not the projection schema and projection is done by processing engines.
  return CloseableIterable.withNoopClose(Iterables.transform(ops.current().snapshots(), snap -> {
    if (snap.manifestListLocation() != null) {
      Expression filter = ignoreResiduals ? Expressions.alwaysTrue() : rowFilter;
      ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(filter);
      return new ManifestListReadTask(ops.io(), table().spec(), new BaseFileScanTask(
          DataFiles.fromManifestList(ops.io().newInputFile(snap.manifestListLocation())),
          schemaString, specString, residuals));
    } else {
      return StaticDataTask.of(
          ops.io().newInputFile(ops.current().metadataFileLocation()),
          snap.allManifests(),
          manifest -> ManifestsTable.manifestFileToRow(table().spec(), manifest));
    }
  }));
}
 
Example #12
Source File: TestResiduals.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testNotIn() {
  Schema schema = new Schema(
      Types.NestedField.optional(50, "dateint", Types.IntegerType.get()),
      Types.NestedField.optional(51, "hour", Types.IntegerType.get())
  );

  PartitionSpec spec = PartitionSpec.builderFor(schema)
      .identity("dateint")
      .build();

  ResidualEvaluator resEval = ResidualEvaluator.of(spec,
      notIn("dateint", 20170815, 20170816, 20170817), true);

  Expression residual = resEval.residualFor(Row.of(20180815));
  Assert.assertEquals("Residual should be alwaysTrue", alwaysTrue(), residual);

  residual = resEval.residualFor(Row.of(20170815));
  Assert.assertEquals("Residual should be alwaysFalse", alwaysFalse(), residual);
}
 
Example #13
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerInclusiveUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 90, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value - 1), Expression.Operation.TRUE);
}
 
Example #14
Source File: Bucket.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate.isUnaryPredicate()) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) {
    return Expressions.predicate(
        predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }

  // comparison predicates can't be projected, notEq can't be projected
  // TODO: small ranges can be projected.
  // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected.
  return null;
}
 
Example #15
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[abcde, abcde]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 
Example #16
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testStringStrict() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value, value + "abc"),
      Expression.Operation.NOT_IN, "[abcde, abcde]");
  assertProjectionStrictValue(spec, in("value", value, value + "abc"), Expression.Operation.FALSE);
}
 
Example #17
Source File: TestDatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString((Integer) v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString((int) literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example #18
Source File: TestDatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testMonthStrictUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-01, 2017-12]");
  assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE);
}
 
Example #19
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerInclusiveLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 100, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 
Example #20
Source File: TestTruncatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testIntegerStrictUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 90, 100]");
  assertProjectionStrictValue(spec, in("value", value, value - 1), Expression.Operation.FALSE);
}
 
Example #21
Source File: TestDatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testYearStrictLowerBound() {
  Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2016-12-31").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", date, anotherDate),
      Expression.Operation.NOT_IN, "[2016, 2017]");
  assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE);
}
 
Example #22
Source File: TestDatesProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testYearInclusiveUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE);

  Integer anotherDate = (Integer) Literal.of("2016-01-01").to(TYPE).value();
  assertProjectionInclusive(spec, in("date", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE);
}
 
Example #23
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example #24
Source File: DataTableScan.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Override
public CloseableIterable<FileScanTask> planFiles(TableOperations ops, Snapshot snapshot,
                                                 Expression rowFilter, boolean ignoreResiduals,
                                                 boolean caseSensitive, boolean colStats) {
  ManifestGroup manifestGroup = new ManifestGroup(ops.io(), snapshot.dataManifests())
      .caseSensitive(caseSensitive)
      .select(colStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS)
      .filterData(rowFilter)
      .specsById(ops.current().specsById())
      .ignoreDeleted();

  if (ignoreResiduals) {
    manifestGroup = manifestGroup.ignoreResiduals();
  }

  if (PLAN_SCANS_WITH_WORKER_POOL && snapshot.dataManifests().size() > 1) {
    manifestGroup = manifestGroup.planWith(ThreadPools.getWorkerPool());
  }

  return manifestGroup.planFiles();
}
 
Example #25
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 
Example #26
Source File: TestTimestampsProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testMonthStrictUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build();

  assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);

  Long anotherDate = (long) Literal.of("2017-11-02T00:00:00.00000").to(TYPE).value();
  assertProjectionStrict(spec, notIn("timestamp", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-11, 2017-12]");
  assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE);
}
 
Example #27
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketIntegerStrict() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 
Example #28
Source File: TestBucketingProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testBucketStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. "abcdefg") is 4
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[4, 9]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 
Example #29
Source File: TestTimestampsProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testYearInclusiveUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build();

  assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE);

  Long anotherDate = (long) Literal.of("2016-12-31T23:59:59.999999").to(TYPE).value();
  assertProjectionInclusive(spec, in("timestamp", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE);
}
 
Example #30
Source File: TestTimestampsProjection.java    From iceberg with Apache License 2.0 6 votes vote down vote up
@Test
public void testHourInclusiveLowerBound() {
  Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build();

  assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09");
  assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01-10");
  assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE);

  Long anotherDate = (long) Literal.of("2016-12-02T00:00:00.00000").to(TYPE).value();
  assertProjectionInclusive(spec, in("timestamp", date, anotherDate),
      Expression.Operation.IN, "[2016-12-02-00, 2017-12-01-10]");
  assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE);
}