org.apache.parquet.filter2.predicate.Operators.DoubleColumn Java Examples

The following examples show how to use org.apache.parquet.filter2.predicate.Operators.DoubleColumn. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ParquetTableSource.java From flink with Apache License 2.0

6 votes

@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #2

Source File: ParquetTableSource.java From flink with Apache License 2.0

6 votes

@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #3

Source File: DictionaryFilterTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testGtEqDouble() throws Exception {
  DoubleColumn d = doubleColumn("double_field");
  double highest = Double.MIN_VALUE;
  for (int value : intValues) {
    highest = Math.max(highest, toDouble(value));
  }

  assertTrue("Should drop: >= highest + 0.00000001",
      canDrop(gtEq(d, highest + 0.00000001), ccmd, dictionaries));
  assertFalse("Should not drop: >= highest",
      canDrop(gtEq(d, highest), ccmd, dictionaries));

  assertFalse("Should not drop: contains matching values",
      canDrop(gtEq(d, Double.MIN_VALUE), ccmd, dictionaries));
}

Example #4

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #5

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #6

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #7

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #8

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #9

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #10

Source File: FilterApi.java From parquet-mr with Apache License 2.0

4 votes

public static DoubleColumn doubleColumn(String columnPath) {
  return new DoubleColumn(ColumnPath.fromDotString(columnPath));
}

Example #11

Source File: TestColumnIndexBuilder.java From parquet-mr with Apache License 2.0

4 votes

@Test
public void testBuildDouble() {
  PrimitiveType type = Types.required(DOUBLE).named("test_double");
  ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  assertThat(builder, instanceOf(DoubleColumnIndexBuilder.class));
  assertNull(builder.build());
  DoubleColumn col = doubleColumn("test_col");

  StatsBuilder sb = new StatsBuilder();
  builder.add(sb.stats(type, -4.2, -4.1));
  builder.add(sb.stats(type, -11.7, 7.0, null));
  builder.add(sb.stats(type, 2.2, 2.2, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 1.9, 2.32));
  builder.add(sb.stats(type, -21.0, 8.1));
  assertEquals(6, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  ColumnIndex columnIndex = builder.build();
  assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 0, 1, 2, 3, 0, 0);
  assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
  assertCorrectValues(columnIndex.getMaxValues(), -4.1, 7.0, 2.2, null, 2.32, 8.1);
  assertCorrectValues(columnIndex.getMinValues(), -4.2, -11.7, 2.2, null, 1.9, -21.0);
  assertCorrectFiltering(columnIndex, eq(col, 0.0), 1, 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
  assertCorrectFiltering(columnIndex, notEq(col, 2.2), 0, 1, 2, 3, 4, 5);
  assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, gt(col, 2.2), 1, 4, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 2.2), 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, lt(col, -4.2), 1, 5);
  assertCorrectFiltering(columnIndex, ltEq(col, -4.2), 0, 1, 5);
  assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 4, 5);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -532.3, -345.2, null, null));
  builder.add(sb.stats(type, -234.7, -234.6, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, -234.6, 2.99999));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, 3.0, 42.83));
  builder.add(sb.stats(type, null, null));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
  assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
  assertCorrectValues(columnIndex.getMaxValues(), null, -345.2, -234.6, null, null, 2.99999, null, 42.83, null);
  assertCorrectValues(columnIndex.getMinValues(), null, -532.3, -234.7, null, null, -234.6, null, 3.0, null);
  assertCorrectFiltering(columnIndex, eq(col, 0.0), 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
  assertCorrectFiltering(columnIndex, notEq(col, 0.0), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
  assertCorrectFiltering(columnIndex, gt(col, 2.99999), 7);
  assertCorrectFiltering(columnIndex, gtEq(col, 2.99999), 5, 7);
  assertCorrectFiltering(columnIndex, lt(col, -234.6), 1, 2);
  assertCorrectFiltering(columnIndex, ltEq(col, -234.6), 1, 2, 5);
  assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 5, 7);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null, null, null, null));
  builder.add(sb.stats(type, 532.3, 345.2));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 234.7, 234.6, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, 234.69, -2.99999));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -3.0, -42.83));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
  assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
  assertCorrectValues(columnIndex.getMaxValues(), null, 532.3, null, 234.7, null, 234.69, null, null, -3.0);
  assertCorrectValues(columnIndex.getMinValues(), null, 345.2, null, 234.6, null, -2.99999, null, null, -42.83);
  assertCorrectFiltering(columnIndex, eq(col, 234.6), 3, 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
  assertCorrectFiltering(columnIndex, notEq(col, 2.2), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
  assertCorrectFiltering(columnIndex, gt(col, 2.2), 1, 3, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 234.69), 1, 3, 5);
  assertCorrectFiltering(columnIndex, lt(col, -2.99999), 8);
  assertCorrectFiltering(columnIndex, ltEq(col, -2.99999), 5, 8);
  assertCorrectFiltering(columnIndex, userDefined(col, DoubleIsInteger.class), 1, 5, 8);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, DoubleIsInteger.class)), 0, 1, 2, 3, 4, 5, 6, 7, 8);
}