Java Code Examples for org.apache.iceberg.expressions.Expressions

The following examples show how to use org.apache.iceberg.expressions.Expressions. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: iceberg   Source File: TestMetadataTableScans.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllEntriesTableHonorsIgnoreResiduals() throws IOException {
  table.newFastAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  Table allEntriesTable = new AllEntriesTable(table.ops(), table);

  TableScan scan1 = allEntriesTable.newScan()
      .filter(Expressions.equal("id", 5));
  validateTaskScanResiduals(scan1, false);

  TableScan scan2 = allEntriesTable.newScan()
      .filter(Expressions.equal("id", 5))
      .ignoreResiduals();
  validateTaskScanResiduals(scan2, true);
}
 
Example 2
Source Project: iceberg   Source File: TestFindFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCaseSensitivity() {
  table.newAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .appendFile(FILE_C)
      .appendFile(FILE_D)
      .commit();

  Iterable<DataFile> files = FindFiles.in(table)
      .caseInsensitive()
      .withMetadataMatching(Expressions.startsWith("FILE_PATH", "/path/to/data-a"))
      .collect();

  Assert.assertEquals(pathSet(FILE_A), pathSet(files));
}
 
Example 3
Source Project: iceberg   Source File: ScanSummary.java    License: Apache License 2.0 6 votes vote down vote up
private void removeTimeFilters(List<Expression> expressions, Expression expression) {
  if (expression.op() == Operation.AND) {
    And and = (And) expression;
    removeTimeFilters(expressions, and.left());
    removeTimeFilters(expressions, and.right());
    return;

  } else if (expression instanceof UnboundPredicate) {
    UnboundPredicate pred = (UnboundPredicate) expression;
    if (pred.term() instanceof NamedReference) {
      NamedReference<?> ref = (NamedReference<?>) pred.term();
      Literal<?> lit = pred.literal();
      if (TIMESTAMP_NAMES.contains(ref.name())) {
        Literal<Long> tsLiteral = lit.to(Types.TimestampType.withoutZone());
        long millis = toMillis(tsLiteral.value());
        addTimestampFilter(Expressions.predicate(pred.op(), "timestamp_ms", millis));
        return;
      }
    }
  }

  expressions.add(expression);
}
 
Example 4
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<Integer> pred) {
  if (pred.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, pred);
  }

  // TODO: for integers, can this return the original predicate?
  // No. the predicate needs to be in terms of the applied value. For all x, apply(x) <= x.
  // Therefore, the lower bound can be transformed outside of a greater-than bound.
  if (pred instanceof BoundUnaryPredicate) {
    return Expressions.predicate(pred.op(), name);
  } else if (pred instanceof BoundLiteralPredicate) {
    return ProjectionUtil.truncateIntegerStrict(name, pred.asLiteralPredicate(), this);
  } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) {
    return ProjectionUtil.transformSet(name, pred.asSetPredicate(), this);
  }
  return null;
}
 
Example 5
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<CharSequence> projectStrict(String name,
                                                    BoundPredicate<CharSequence> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate instanceof BoundUnaryPredicate) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate instanceof BoundLiteralPredicate) {
    BoundLiteralPredicate<CharSequence> pred = predicate.asLiteralPredicate();
    if (pred.op() == Expression.Operation.STARTS_WITH) {
      if (pred.literal().value().length() < width()) {
        return Expressions.predicate(pred.op(), name, pred.literal().value());
      } else if (pred.literal().value().length() == width()) {
        return Expressions.equal(name, pred.literal().value());
      }
    } else {
      return ProjectionUtil.truncateArrayStrict(name, pred, this);
    }
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.NOT_IN) {
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }
  return null;
}
 
Example 6
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<ByteBuffer> project(String name,
                                            BoundPredicate<ByteBuffer> pred) {
  if (pred.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, pred);
  }

  if (pred.isUnaryPredicate()) {
    return Expressions.predicate(pred.op(), name);
  } else if (pred.isLiteralPredicate()) {
    return ProjectionUtil.truncateArray(name, pred.asLiteralPredicate(), this);
  } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) {
    return ProjectionUtil.transformSet(name, pred.asSetPredicate(), this);
  }
  return null;
}
 
Example 7
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<ByteBuffer> projectStrict(String name,
                                                  BoundPredicate<ByteBuffer> pred) {
  if (pred.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, pred);
  }

  if (pred.isUnaryPredicate()) {
    return Expressions.predicate(pred.op(), name);
  } else if (pred.isLiteralPredicate()) {
    return ProjectionUtil.truncateArrayStrict(name, pred.asLiteralPredicate(), this);
  } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) {
    return ProjectionUtil.transformSet(name, pred.asSetPredicate(), this);
  }
  return null;
}
 
Example 8
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<BigDecimal> project(String name,
                                            BoundPredicate<BigDecimal> pred) {
  if (pred.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, pred);
  }

  if (pred.isUnaryPredicate()) {
    return Expressions.predicate(pred.op(), name);
  } else if (pred.isLiteralPredicate()) {
    return ProjectionUtil.truncateDecimal(name, pred.asLiteralPredicate(), this);
  } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) {
    return ProjectionUtil.transformSet(name, pred.asSetPredicate(), this);
  }
  return null;
}
 
Example 9
Source Project: iceberg   Source File: Bucket.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate.isUnaryPredicate()) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) {
    return Expressions.predicate(
        predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.IN) { // notIn can't be projected
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }

  // comparison predicates can't be projected, notEq can't be projected
  // TODO: small ranges can be projected.
  // for example, (x > 0) and (x < 3) can be turned into in({1, 2}) and projected.
  return null;
}
 
Example 10
Source Project: iceberg   Source File: Bucket.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<T> predicate) {
  if (predicate.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, predicate);
  }

  if (predicate.isUnaryPredicate()) {
    return Expressions.predicate(predicate.op(), name);
  } else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.NOT_EQ) {
    // TODO: need to translate not(eq(...)) into notEq in expressions
    return Expressions.predicate(predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
  } else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.NOT_IN) {
    return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
  }

  // no strict projection for comparison or equality
  return null;
}
 
Example 11
Source Project: iceberg   Source File: TestResiduals.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnpartitionedResiduals() {
  Expression[] expressions = new Expression[] {
      Expressions.alwaysTrue(),
      Expressions.alwaysFalse(),
      Expressions.lessThan("a", 5),
      Expressions.greaterThanOrEqual("b", 16),
      Expressions.notNull("c"),
      Expressions.isNull("d"),
      Expressions.in("e", 1, 2, 3),
      Expressions.notIn("f", 1, 2, 3)
  };

  for (Expression expr : expressions) {
    ResidualEvaluator residualEvaluator = ResidualEvaluator.of(PartitionSpec.unpartitioned(), expr, true);
    Assert.assertEquals("Should return expression",
        expr, residualEvaluator.residualFor(Row.of()));
  }
}
 
Example 12
Source Project: iceberg   Source File: TestLocalScan.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testProjectWithMissingFilterColumn() {
  Iterable<Record> results = IcebergGenerics.read(sharedTable)
      .where(Expressions.greaterThanOrEqual("id", 1))
      .where(Expressions.lessThan("id", 21))
      .select("data").build();

  Set<String> expected = Sets.newHashSet();
  for (Record record : concat(file1Records, file2Records, file3Records)) {
    Long id = (Long) record.getField("id");
    if (id >= 1 && id < 21) {
      expected.add(record.getField("data").toString());
    }
  }

  results.forEach(record ->
      Assert.assertEquals("Record should have two projected fields", 2, record.size()));

  Assert.assertEquals("Should project correct rows",
      expected,
      Sets.newHashSet(transform(results, record -> record.getField("data").toString())));
}
 
Example 13
Source Project: iceberg   Source File: TestIcebergInputFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFilterExp() throws Exception {
  File location = temp.newFolder(format.name());
  Assert.assertTrue(location.delete());
  Table table = tables.create(SCHEMA, SPEC,
                              ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, format.name()),
                              location.toString());
  List<Record> expectedRecords = RandomGenericData.generate(table.schema(), 2, 0L);
  expectedRecords.get(0).set(2, "2020-03-20");
  expectedRecords.get(1).set(2, "2020-03-20");
  DataFile dataFile1 = writeFile(table, Row.of("2020-03-20", 0), format, expectedRecords);
  DataFile dataFile2 = writeFile(table, Row.of("2020-03-21", 0), format,
                                 RandomGenericData.generate(table.schema(), 2, 0L));
  table.newAppend()
       .appendFile(dataFile1)
       .appendFile(dataFile2)
       .commit();
  Job job = Job.getInstance(conf);
  IcebergInputFormat.ConfigBuilder configBuilder = IcebergInputFormat.configure(job);
  configBuilder.readFrom(location.toString())
               .filter(Expressions.equal("date", "2020-03-20"));
  validate(job, expectedRecords);
}
 
Example 14
Source Project: iceberg   Source File: TestFindFiles.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWithRecordsMatching() {
  table.newAppend()
      .appendFile(DataFiles.builder(SPEC)
          .withInputFile(Files.localInput("/path/to/data-e.parquet"))
          .withPartitionPath("data_bucket=4")
          .withMetrics(new Metrics(3L,
              null, // no column sizes
              ImmutableMap.of(1, 3L), // value count
              ImmutableMap.of(1, 0L), // null count
              ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1)),  // lower bounds
              ImmutableMap.of(1, Conversions.toByteBuffer(Types.IntegerType.get(), 5)))) // lower bounds
          .build())
      .commit();

  final Iterable<DataFile> files = FindFiles.in(table)
      .withRecordsMatching(Expressions.equal("id", 1))
      .collect();

  Assert.assertEquals(Sets.newHashSet("/path/to/data-e.parquet"), pathSet(files));
}
 
Example 15
Source Project: iceberg   Source File: IcebergStorage.java    License: Apache License 2.0 6 votes vote down vote up
private org.apache.iceberg.expressions.Expression convert(OpType op, Column col, Const constant) {
  String name = col.getName();
  Object value = constant.getValue();

  switch (op) {
    case OP_GE: return Expressions.greaterThanOrEqual(name, value);
    case OP_GT: return Expressions.greaterThan(name, value);
    case OP_LE: return Expressions.lessThanOrEqual(name, value);
    case OP_LT: return Expressions.lessThan(name, value);
    case OP_EQ: return Expressions.equal(name, value);
    case OP_NE: return Expressions.notEqual(name, value);
  }

  throw new RuntimeException(
      String.format("[%s]: Failed to pushdown expression: %s %s %s", signature, col, op, constant));
}
 
Example 16
Source Project: iceberg   Source File: TestMetadataTableScans.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAllManifestsTableHonorsIgnoreResiduals() throws IOException {
  table.newFastAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  Table allManifestsTable = new AllManifestsTable(table.ops(), table);

  TableScan scan1 = allManifestsTable.newScan()
      .filter(Expressions.equal("id", 5));
  validateTaskScanResiduals(scan1, false);

  TableScan scan2 = allManifestsTable.newScan()
      .filter(Expressions.equal("id", 5))
      .ignoreResiduals();
  validateTaskScanResiduals(scan2, true);
}
 
Example 17
Source Project: iceberg   Source File: TestFilterFiles.java    License: Apache License 2.0 6 votes vote down vote up
private void testCaseInsensitiveFilterFiles(Table table) {
  Map<Integer, ByteBuffer> lowerBounds = new HashMap<>();
  Map<Integer, ByteBuffer> upperBounds = new HashMap<>();
  lowerBounds.put(1, Conversions.toByteBuffer(Types.IntegerType.get(), 1));
  upperBounds.put(1, Conversions.toByteBuffer(Types.IntegerType.get(), 2));

  Metrics metrics = new Metrics(2L, Maps.newHashMap(), Maps.newHashMap(),
      Maps.newHashMap(), lowerBounds, upperBounds);

  DataFile file = DataFiles.builder(table.spec())
      .withPath("/path/to/file.parquet")
      .withFileSizeInBytes(0)
      .withMetrics(metrics)
      .build();

  table.newAppend().appendFile(file).commit();

  table.refresh();

  TableScan emptyScan = table.newScan().caseSensitive(false).filter(Expressions.equal("ID", 5));
  assertEquals(0, Iterables.size(emptyScan.planFiles()));

  TableScan nonEmptyScan = table.newScan().caseSensitive(false).filter(Expressions.equal("ID", 1));
  assertEquals(1, Iterables.size(nonEmptyScan.planFiles()));
}
 
Example 18
Source Project: iceberg   Source File: TestMetadataTableScans.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testManifestsTableAlwaysIgnoresResiduals() throws IOException {
  table.newFastAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  Table manifestsTable = new ManifestsTable(table.ops(), table);

  TableScan scan = manifestsTable.newScan()
      .filter(Expressions.equal("id", 5));

  try (CloseableIterable<FileScanTask> tasks = scan.planFiles()) {
    Assert.assertTrue("Tasks should not be empty", Iterables.size(tasks) > 0);
    for (FileScanTask task : tasks) {
      Assert.assertEquals("Residuals must be ignored", Expressions.alwaysTrue(), task.residual());
    }
  }
}
 
Example 19
Source Project: iceberg   Source File: DataFilesTable.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected CloseableIterable<FileScanTask> planFiles(
    TableOperations ops, Snapshot snapshot, Expression rowFilter,
    boolean ignoreResiduals, boolean caseSensitive, boolean colStats) {
  CloseableIterable<ManifestFile> manifests = CloseableIterable.withNoopClose(snapshot.dataManifests());
  String schemaString = SchemaParser.toJson(schema());
  String specString = PartitionSpecParser.toJson(PartitionSpec.unpartitioned());
  Expression filter = ignoreResiduals ? Expressions.alwaysTrue() : rowFilter;
  ResidualEvaluator residuals = ResidualEvaluator.unpartitioned(filter);

  // Data tasks produce the table schema, not the projection schema and projection is done by processing engines.
  // This data task needs to use the table schema, which may not include a partition schema to avoid having an
  // empty struct in the schema for unpartitioned tables. Some engines, like Spark, can't handle empty structs in
  // all cases.
  return CloseableIterable.transform(manifests, manifest ->
      new ManifestReadTask(ops.io(), manifest, fileSchema, schemaString, specString, residuals));
}
 
Example 20
Source Project: iceberg   Source File: TestMetadataTableScans.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testDataFilesTableHonorsIgnoreResiduals() throws IOException {
  table.newFastAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  Table dataFilesTable = new DataFilesTable(table.ops(), table);

  TableScan scan1 = dataFilesTable.newScan()
      .filter(Expressions.equal("id", 5));
  validateTaskScanResiduals(scan1, false);

  TableScan scan2 = dataFilesTable.newScan()
      .filter(Expressions.equal("id", 5))
      .ignoreResiduals();
  validateTaskScanResiduals(scan2, true);
}
 
Example 21
Source Project: iceberg   Source File: ManifestReader.java    License: Apache License 2.0 6 votes vote down vote up
CloseableIterable<ManifestEntry<F>> entries() {
  if ((rowFilter != null && rowFilter != Expressions.alwaysTrue()) ||
      (partFilter != null && partFilter != Expressions.alwaysTrue())) {
    Evaluator evaluator = evaluator();
    InclusiveMetricsEvaluator metricsEvaluator = metricsEvaluator();

    // ensure stats columns are present for metrics evaluation
    boolean requireStatsProjection = requireStatsProjection(rowFilter, columns);
    Collection<String> projectColumns = requireStatsProjection ? withStatsColumns(columns) : columns;

    return CloseableIterable.filter(
        open(projection(fileSchema, fileProjection, projectColumns, caseSensitive)),
        entry -> entry != null &&
            evaluator.eval(entry.file().partition()) &&
            metricsEvaluator.eval(entry.file()));
  } else {
    return open(projection(fileSchema, fileProjection, columns, caseSensitive));
  }
}
 
Example 22
Source Project: iceberg   Source File: OrcIterable.java    License: Apache License 2.0 5 votes vote down vote up
OrcIterable(InputFile file, Configuration config, Schema schema,
            Long start, Long length,
            Function<TypeDescription, OrcRowReader<?>> readerFunction, boolean caseSensitive, Expression filter) {
  this.schema = schema;
  this.readerFunction = readerFunction;
  this.file = file;
  this.start = start;
  this.length = length;
  this.config = config;
  this.caseSensitive = caseSensitive;
  this.filter = (filter == Expressions.alwaysTrue()) ? null : filter;
}
 
Example 23
Source Project: iceberg   Source File: ProjectionUtil.java    License: Apache License 2.0 5 votes vote down vote up
private static <T> UnboundPredicate<T> removeTransform(String partitionName, BoundPredicate<T> pred) {
  if (pred.isUnaryPredicate()) {
    return Expressions.predicate(pred.op(), partitionName);
  } else if (pred.isLiteralPredicate()) {
    return Expressions.predicate(pred.op(), partitionName, pred.asLiteralPredicate().literal());
  } else if (pred.isSetPredicate()) {
    return Expressions.predicate(pred.op(), partitionName, pred.asSetPredicate().literalSet());
  }
  throw new UnsupportedOperationException("Cannot replace transform in unknown predicate: " + pred);
}
 
Example 24
Source Project: metacat   Source File: IcebergFilterGenerator.java    License: Apache License 2.0 5 votes vote down vote up
private Expression evalSingleTerm(final ASTCOMPARE node, final Object data) {
    final Object value = node.jjtGetChild(0).jjtAccept(this, data);
    if (value != null) {
        return Boolean.parseBoolean(value.toString())
            ? Expressions.alwaysTrue() : Expressions.alwaysFalse();
    }
    return Expressions.alwaysFalse();
}
 
Example 25
Source Project: iceberg   Source File: Truncate.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public UnboundPredicate<Long> project(String name, BoundPredicate<Long> pred) {
  if (pred.term() instanceof BoundTransform) {
    return ProjectionUtil.projectTransformPredicate(this, name, pred);
  }

  if (pred.isUnaryPredicate()) {
    return Expressions.predicate(pred.op(), name);
  } else if (pred.isLiteralPredicate()) {
    return ProjectionUtil.truncateLong(name, pred.asLiteralPredicate(), this);
  } else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) {
    return ProjectionUtil.transformSet(name, pred.asSetPredicate(), this);
  }
  return null;
}
 
Example 26
Source Project: metacat   Source File: IcebergFilterGenerator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Object visit(final ASTBETWEEN node, final Object data) {
    final Object value = node.jjtGetChild(0).jjtAccept(this, data);
    final Object startValue = node.jjtGetChild(1).jjtAccept(this, data);
    final Object endValue = node.jjtGetChild(2).jjtAccept(this, data);
    final Expression compare1 =
        createIcebergExpression(value, startValue, node.not ? Compare.LT : Compare.GTE);
    final Expression compare2 =
        createIcebergExpression(value, endValue, node.not ? Compare.GT : Compare.LTE);
    return (node.not)
        ? Expressions.or(compare1, compare2) : Expressions.and(compare1, compare2);
}
 
Example 27
Source Project: iceberg   Source File: TestManifestCleanup.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOverwrite() {
  Assert.assertEquals("Table should start with no manifests",
      0, listManifestFiles().size());

  table.newAppend()
      .appendFile(FILE_A)
      .appendFile(FILE_B)
      .commit();

  Assert.assertEquals("Table should have one append manifest",
      1, table.currentSnapshot().allManifests().size());

  table.newOverwrite()
      .overwriteByRowFilter(Expressions.alwaysTrue())
      .addFile(FILE_C)
      .addFile(FILE_D)
      .commit();

  Assert.assertEquals("Table should have one delete manifest and one append manifest",
      2, table.currentSnapshot().allManifests().size());

  table.newOverwrite()
      .overwriteByRowFilter(Expressions.alwaysTrue())
      .addFile(FILE_A)
      .addFile(FILE_B)
      .commit();

  Assert.assertEquals("Table should have one delete manifest and one append manifest",
      2, table.currentSnapshot().allManifests().size());
}
 
Example 28
Source Project: iceberg   Source File: TestProjection.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testIdentityProjection() {
  List<UnboundPredicate<?>> predicates = Lists.newArrayList(
      Expressions.notNull("id"),
      Expressions.isNull("id"),
      Expressions.lessThan("id", 100),
      Expressions.lessThanOrEqual("id", 101),
      Expressions.greaterThan("id", 102),
      Expressions.greaterThanOrEqual("id", 103),
      Expressions.equal("id", 104),
      Expressions.notEqual("id", 105)
  );

  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
      .identity("id")
      .build();

  for (UnboundPredicate<?> predicate : predicates) {
    // get the projected predicate
    Expression expr = Projections.inclusive(spec).project(predicate);
    UnboundPredicate<?> projected = assertAndUnwrapUnbound(expr);

    // check inclusive the bound predicate to ensure the types are correct
    BoundPredicate<?> bound = assertAndUnwrap(predicate.bind(spec.schema().asStruct(), true));

    Assert.assertEquals("Field name should match partition struct field",
        "id", projected.ref().name());
    Assert.assertEquals("Operation should match", bound.op(), projected.op());

    if (bound.isLiteralPredicate()) {
      Assert.assertEquals("Literal should be equal",
          bound.asLiteralPredicate().literal().value(), projected.literal().value());
    } else {
      Assert.assertNull("Literal should be null", projected.literal());
    }
  }
}
 
Example 29
Source Project: iceberg   Source File: TestProjection.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCaseInsensitiveIdentityProjection() {
  List<UnboundPredicate<?>> predicates = Lists.newArrayList(
      Expressions.notNull("ID"),
      Expressions.isNull("ID"),
      Expressions.lessThan("ID", 100),
      Expressions.lessThanOrEqual("ID", 101),
      Expressions.greaterThan("ID", 102),
      Expressions.greaterThanOrEqual("ID", 103),
      Expressions.equal("ID", 104),
      Expressions.notEqual("ID", 105)
  );

  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
      .identity("id")
      .build();

  for (UnboundPredicate<?> predicate : predicates) {
    // get the projected predicate
    Expression expr = Projections.inclusive(spec, false).project(predicate);
    UnboundPredicate<?> projected = assertAndUnwrapUnbound(expr);

    // check inclusive the bound predicate to ensure the types are correct
    BoundPredicate<?> bound = assertAndUnwrap(predicate.bind(spec.schema().asStruct(), false));

    Assert.assertEquals("Field name should match partition struct field",
        "id", projected.ref().name());
    Assert.assertEquals("Operation should match", bound.op(), projected.op());

    if (bound.isLiteralPredicate()) {
      Assert.assertEquals("Literal should be equal",
          bound.asLiteralPredicate().literal().value(), projected.literal().value());
    } else {
      Assert.assertNull("Literal should be null", projected.literal());
    }
  }
}
 
Example 30
Source Project: iceberg   Source File: TestProjection.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCaseSensitiveIdentityProjection() {
  PartitionSpec spec = PartitionSpec.builderFor(SCHEMA)
      .identity("id")
      .build();

  assertThrows(
      "X != x when case sensitivity is on",
      ValidationException.class,
      "Cannot find field 'ID' in struct",
      () -> Projections.inclusive(spec, true).project(Expressions.notNull("ID")));
}