org.apache.iceberg.expressions.Expression Java Examples

public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 

static <T> UnboundPredicate<T> truncateLong(
    String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
  long boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      // adjust closed and then transform ltEq
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary - 1L));
    case LT_EQ:
      return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary));
    case GT:
      // adjust closed and then transform gtEq
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary + 1L));
    case GT_EQ:
      return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary));
    case EQ:
      return predicate(pred.op(), name, transform.apply(boundary));
    default:
      return null;
  }
}
 

static <T> UnboundPredicate<T> truncateIntegerStrict(
    String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) {
  int boundary = pred.literal().value();
  switch (pred.op()) {
    case LT:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary));
    case LT_EQ:
      return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1));
    case GT:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary));
    case GT_EQ:
      return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1));
    case NOT_EQ:
      return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
    case EQ:
      // there is no predicate that guarantees equality because adjacent ints transform to the same value
      return null;
    default:
      return null;
  }
}
 

/**
 * Filter results to files in any one of the given partitions.
 *
 * @param spec a spec for the partitions
 * @param partitions a list of StructLike that stores a partition tuple
 * @return this for method chaining
 */
public Builder inPartitions(PartitionSpec spec, List<StructLike> partitions) {
  Preconditions.checkArgument(spec.equals(ops.current().spec(spec.specId())),
      "Partition spec does not belong to table: %s", table);

  Expression partitionSetFilter = Expressions.alwaysFalse();
  for (StructLike partitionData : partitions) {
    Expression partFilter = Expressions.alwaysTrue();
    for (int i = 0; i < spec.fields().size(); i += 1) {
      PartitionField field = spec.fields().get(i);
      partFilter = Expressions.and(
          partFilter,
          Expressions.equal(field.name(), partitionData.get(i, Object.class)));
    }
    partitionSetFilter = Expressions.or(partitionSetFilter, partFilter);
  }

  if (partitionFilter != Expressions.alwaysTrue()) {
    this.partitionFilter = Expressions.or(partitionFilter, partitionSetFilter);
  } else {
    this.partitionFilter = partitionSetFilter;
  }

  return this;
}
 

@Test
public void testIntegerStrictLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 100, 100]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 

@Test
public void testBinaryStrict() throws Exception {
  ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
  Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
  String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue);
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue);
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue);
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  ByteBuffer anotherValue = ByteBuffer.wrap("abcdehij".getBytes("UTF-8"));
  assertProjectionStrict(spec, notIn("value", value, anotherValue),
      Expression.Operation.NOT_IN, String.format("[%s, %s]", expectedValue, expectedValue));
  assertProjectionStrictValue(spec, in("value", value, anotherValue), Expression.Operation.FALSE);
}
 

@Test
public void testBucketLongStrict() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 

@Test
public void testDecimalStrictLowerBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
          Expression.Operation.NOT_IN, "[99.00, 100.00, 101.00]");
  assertProjectionStrictValue(spec, in("value", value, value.add(delta)), Expression.Operation.FALSE);
}
 

@Test
public void testBucketLongInclusive() {
  Long value = 100L;
  Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "6");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[6, 7, 8]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 

@Test
public void testDecimalStrictUpperBound() {
  Types.DecimalType type = Types.DecimalType.of(9, 2);
  BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
  Schema schema = new Schema(optional(1, "value", type));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  BigDecimal delta = new BigDecimal(1);
  assertProjectionStrict(spec, notIn("value", value.add(delta), value, value.subtract(delta)),
      Expression.Operation.NOT_IN, "[98.90, 99.90, 100.90]");
  assertProjectionStrictValue(spec, in("value", value, value.subtract(delta)), Expression.Operation.FALSE);
}
 

@Override
protected CloseableIterable<FileScanTask> planFiles(
    TableOperations ops, Snapshot snapshot, Expression rowFilter,
    boolean ignoreResiduals, boolean caseSensitive, boolean colStats) {
  String schemaString = SchemaParser.toJson(schema());
  String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned());

  // Data tasks produce the table schema, not the projection schema and projection is done by processing engines.
  return CloseableIterable.withNoopClose(Iterables.transform(ops.current().snapshots(), snap -> {
    if (snap.manifestListLocation() != null) {
      Expression filter = ignoreResiduals ? Expressions.alwaysTrue() : rowFilter;
      ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(filter);
      return new ManifestListReadTask(ops.io(), table().spec(), new BaseFileScanTask(
          DataFiles.fromManifestList(ops.io().newInputFile(snap.manifestListLocation())),
          schemaString, specString, residuals));
    } else {
      return StaticDataTask.of(
          ops.io().newInputFile(ops.current().metadataFileLocation()),
          snap.allManifests(),
          manifest -> ManifestsTable.manifestFileToRow(table().spec(), manifest));
    }
  }));
}
 

@Test
public void testNotIn() {
  Schema schema = new Schema(
      Types.NestedField.optional(50, "dateint", Types.IntegerType.get()),
      Types.NestedField.optional(51, "hour", Types.IntegerType.get())
  );

  PartitionSpec spec = PartitionSpec.builderFor(schema)
      .identity("dateint")
      .build();

  ResidualEvaluator resEval = ResidualEvaluator.of(spec,
      notIn("dateint", 20170815, 20170816, 20170817), true);

  Expression residual = resEval.residualFor(Row.of(20180815));
  Assert.assertEquals("Residual should be alwaysTrue", alwaysTrue(), residual);

  residual = resEval.residualFor(Row.of(20170815));
  Assert.assertEquals("Residual should be alwaysFalse", alwaysFalse(), residual);
}
 

@Test
public void testIntegerInclusiveUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 90, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value - 1), Expression.Operation.TRUE);
}
 

@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate.isUnaryPredicate()) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) {
    return Expressions.predicate(
        predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }

  // comparison predicates can't be projected, notEq can't be projected
  // TODO: small ranges can be projected.
  // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected.
  return null;
}
 

@Test
public void testStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[abcde, abcde]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 

@Test
public void testStringStrict() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value, value + "abc"),
      Expression.Operation.NOT_IN, "[abcde, abcde]");
  assertProjectionStrictValue(spec, in("value", value, value + "abc"), Expression.Operation.FALSE);
}
 

public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString((Integer) v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString((int) literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 

@Test
public void testMonthStrictUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-01, 2017-12]");
  assertProjectionStrictValue(spec, in("date", anotherDate, date), Expression.Operation.FALSE);
}
 

@Test
public void testIntegerInclusiveLowerBound() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
  assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100");
  assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100");
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value - 1, value, value + 1),
      Expression.Operation.IN, "[90, 100, 100]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + 1), Expression.Operation.TRUE);
}
 

@Test
public void testIntegerStrictUpperBound() {
  Integer value = 99;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();

  assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
  assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
  assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[90, 90, 100]");
  assertProjectionStrictValue(spec, in("value", value, value - 1), Expression.Operation.FALSE);
}
 

@Test
public void testYearStrictLowerBound() {
  Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017");
  assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017");
  assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016");
  assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017");
  assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);

  Integer anotherDate = (Integer) Literal.of("2016-12-31").to(TYPE).value();
  assertProjectionStrict(spec, notIn("date", date, anotherDate),
      Expression.Operation.NOT_IN, "[2016, 2017]");
  assertProjectionStrictValue(spec, in("date", date, anotherDate), Expression.Operation.FALSE);
}
 

@Test
public void testYearInclusiveUpperBound() {
  Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();

  assertProjectionInclusive(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("date", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("date", date), Expression.Operation.TRUE);

  Integer anotherDate = (Integer) Literal.of("2016-01-01").to(TYPE).value();
  assertProjectionInclusive(spec, in("date", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("date", date, anotherDate), Expression.Operation.TRUE);
}
 

public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
                                      Expression.Operation expectedOp, String expectedLiteral) {
  Expression projection = Projections.inclusive(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(predicate.op(), expectedOp);

  Assert.assertNotEquals("Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 

@Override
public CloseableIterable<FileScanTask> planFiles(TableOperations ops, Snapshot snapshot,
                                                 Expression rowFilter, boolean ignoreResiduals,
                                                 boolean caseSensitive, boolean colStats) {
  ManifestGroup manifestGroup = new ManifestGroup(ops.io(), snapshot.dataManifests())
      .caseSensitive(caseSensitive)
      .select(colStats ? SCAN_WITH_STATS_COLUMNS : SCAN_COLUMNS)
      .filterData(rowFilter)
      .specsById(ops.current().specsById())
      .ignoreDeleted();

  if (ignoreResiduals) {
    manifestGroup = manifestGroup.ignoreResiduals();
  }

  if (PLAN_SCANS_WITH_WORKER_POOL && snapshot.dataManifests().size() > 1) {
    manifestGroup = manifestGroup.planWith(ThreadPools.getWorkerPool());
  }

  return manifestGroup.planFiles();
}
 

public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
                                   Expression.Operation expectedOp, String expectedLiteral) {

  Expression projection = Projections.strict(spec).project(filter);
  UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);

  Assert.assertEquals(expectedOp, predicate.op());

  Assert.assertNotEquals("Strict projection never runs for IN", Expression.Operation.IN, predicate.op());

  Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
  if (predicate.op() == Expression.Operation.NOT_IN) {
    Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
    String actual = Lists.newArrayList(values).stream().sorted()
        .map(v -> transform.toHumanString(v)).collect(Collectors.toList()).toString();
    Assert.assertEquals(expectedLiteral, actual);
  } else {
    Literal literal = predicate.literal();
    String output = transform.toHumanString(literal.value());
    Assert.assertEquals(expectedLiteral, output);
  }
}
 

@Test
public void testMonthStrictUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build();

  assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12");
  assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018-01");
  assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12");
  assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12");
  assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);

  Long anotherDate = (long) Literal.of("2017-11-02T00:00:00.00000").to(TYPE).value();
  assertProjectionStrict(spec, notIn("timestamp", anotherDate, date),
      Expression.Operation.NOT_IN, "[2017-11, 2017-12]");
  assertProjectionStrictValue(spec, in("timestamp", anotherDate, date), Expression.Operation.FALSE);
}
 

@Test
public void testBucketIntegerStrict() {
  Integer value = 100;
  Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. 100) is 6
  assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "6");
  assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, lessThanOrEqual("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThan("value", value), Expression.Operation.FALSE);
  assertProjectionStrictValue(spec, greaterThanOrEqual("value", value), Expression.Operation.FALSE);

  assertProjectionStrict(spec, notIn("value", value - 1, value, value + 1),
      Expression.Operation.NOT_IN, "[6, 7, 8]");
  assertProjectionStrictValue(spec, in("value", value, value + 1), Expression.Operation.FALSE);
}
 

@Test
public void testBucketStringInclusive() {
  String value = "abcdefg";
  Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
  PartitionSpec spec = PartitionSpec.builderFor(schema).bucket("value", 10).build();

  // the bucket number of the value (i.e. "abcdefg") is 4
  assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "4");
  assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, lessThanOrEqual("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThan("value", value), Expression.Operation.TRUE);
  assertProjectionInclusiveValue(spec, greaterThanOrEqual("value", value), Expression.Operation.TRUE);

  assertProjectionInclusive(spec, in("value", value, value + "abc"),
      Expression.Operation.IN, "[4, 9]");
  assertProjectionInclusiveValue(spec, notIn("value", value, value + "abc"), Expression.Operation.TRUE);
}
 

@Test
public void testYearInclusiveUpperBound() {
  Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build();

  assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017");
  assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018");
  assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017");
  assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017");
  assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE);

  Long anotherDate = (long) Literal.of("2016-12-31T23:59:59.999999").to(TYPE).value();
  assertProjectionInclusive(spec, in("timestamp", date, anotherDate),
      Expression.Operation.IN, "[2016, 2017]");
  assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE);
}
 

@Test
public void testHourInclusiveLowerBound() {
  Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value();
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build();

  assertProjectionInclusive(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09");
  assertProjectionInclusive(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10");
  assertProjectionInclusive(spec, equal("timestamp", date), Expression.Operation.EQ, "2017-12-01-10");
  assertProjectionInclusiveValue(spec, notEqual("timestamp", date), Expression.Operation.TRUE);

  Long anotherDate = (long) Literal.of("2016-12-02T00:00:00.00000").to(TYPE).value();
  assertProjectionInclusive(spec, in("timestamp", date, anotherDate),
      Expression.Operation.IN, "[2016-12-02-00, 2017-12-01-10]");
  assertProjectionInclusiveValue(spec, notIn("timestamp", date, anotherDate), Expression.Operation.TRUE);
}