org.apache.parquet.filter2.predicate.Operators.LongColumn Java Examples

The following examples show how to use org.apache.parquet.filter2.predicate.Operators.LongColumn. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: ParquetTableSource.java From flink with Apache License 2.0

6 votes

@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #2

Source File: ParquetTableSource.java From flink with Apache License 2.0

6 votes

@Nullable
private FilterPredicate lessThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThan, "exp has to be LessThan");

	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.lt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.lt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.lt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.lt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #3

Source File: DictionaryFilterTest.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testLtEqLong() throws Exception {
  LongColumn i64 = longColumn("int64_field");
  long lowest = Long.MAX_VALUE;
  for (long value : longValues) {
    lowest = Math.min(lowest, value);
  }

  assertTrue("Should drop: <= lowest - 1",
      canDrop(ltEq(i64, lowest - 1), ccmd, dictionaries));
  assertFalse("Should not drop: <= lowest",
      canDrop(ltEq(i64, lowest), ccmd, dictionaries));

  assertFalse("Should not drop: contains matching values",
      canDrop(ltEq(i64, Long.MAX_VALUE), ccmd, dictionaries));
}

Example #4

Source File: TestRecordLevelFilters.java From parquet-mr with Apache License 2.0

6 votes

@Test
public void testUserDefinedByInstance() throws Exception {
  LongColumn name = longColumn("id");

  final HashSet<Long> h = new HashSet<Long>();
  h.add(20L); 
  h.add(27L);
  h.add(28L);
  
  FilterPredicate pred = userDefined(name, new SetInFilter(h));

  List<Group> found = PhoneBookWriter.readFile(phonebookFile, FilterCompat.get(pred));

  assertFilter(found, new UserFilter() {
    @Override
    public boolean keep(User u) {
      return u != null && h.contains(u.getId());
    }
  });
}

Example #5

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #6

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #7

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #8

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThan(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThan, "exp has to be GreaterThan");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gt((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gt((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gt((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gt((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #9

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate greaterThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof GreaterThanOrEqual, "exp has to be GreaterThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.gtEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.gtEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.gtEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.gtEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #10

Source File: ParquetTableSource.java From flink with Apache License 2.0

5 votes

@Nullable
private FilterPredicate lessThanOrEqual(Expression exp, Tuple2<Column, Comparable> columnPair) {
	Preconditions.checkArgument(exp instanceof LessThanOrEqual, "exp has to be LessThanOrEqual");
	if (columnPair.f0 instanceof IntColumn) {
		return FilterApi.ltEq((IntColumn) columnPair.f0, (Integer) columnPair.f1);
	} else if (columnPair.f0 instanceof LongColumn) {
		return FilterApi.ltEq((LongColumn) columnPair.f0, (Long) columnPair.f1);
	} else if (columnPair.f0 instanceof DoubleColumn) {
		return FilterApi.ltEq((DoubleColumn) columnPair.f0, (Double) columnPair.f1);
	} else if (columnPair.f0 instanceof FloatColumn) {
		return FilterApi.ltEq((FloatColumn) columnPair.f0, (Float) columnPair.f1);
	}

	return null;
}

Example #11

Source File: FilterApi.java From parquet-mr with Apache License 2.0

4 votes

public static LongColumn longColumn(String columnPath) {
  return new LongColumn(ColumnPath.fromDotString(columnPath));
}

Example #12

Source File: TestColumnIndexBuilder.java From parquet-mr with Apache License 2.0

4 votes

@Test
public void testBuildInt64() {
  PrimitiveType type = Types.required(INT64).named("test_int64");
  ColumnIndexBuilder builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  assertThat(builder, instanceOf(LongColumnIndexBuilder.class));
  assertNull(builder.build());
  LongColumn col = longColumn("test_col");

  StatsBuilder sb = new StatsBuilder();
  builder.add(sb.stats(type, -4l, 10l));
  builder.add(sb.stats(type, -11l, 7l, null));
  builder.add(sb.stats(type, 2l, 2l, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 1l, 2l));
  builder.add(sb.stats(type, -21l, 8l));
  assertEquals(6, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  ColumnIndex columnIndex = builder.build();
  assertEquals(BoundaryOrder.UNORDERED, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 0l, 1l, 2l, 3l, 0l, 0l);
  assertCorrectNullPages(columnIndex, false, false, false, true, false, false);
  assertCorrectValues(columnIndex.getMaxValues(), 10l, 7l, 2l, null, 2l, 8l);
  assertCorrectValues(columnIndex.getMinValues(), -4l, -11l, 2l, null, 1l, -21l);
  assertCorrectFiltering(columnIndex, eq(col, 0l), 0, 1, 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 1, 2, 3);
  assertCorrectFiltering(columnIndex, notEq(col, 0l), 0, 1, 2, 3, 4, 5);
  assertCorrectFiltering(columnIndex, notEq(col, null), 0, 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, gt(col, 2l), 0, 1, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 2l), 0, 1, 2, 4, 5);
  assertCorrectFiltering(columnIndex, lt(col, -21l));
  assertCorrectFiltering(columnIndex, ltEq(col, -21l), 5);
  assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 0, 1, 5);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -532l, -345l, null, null));
  builder.add(sb.stats(type, -234l, -42l, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, -42l, 2l));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -3l, 42l));
  builder.add(sb.stats(type, null, null));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.ASCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 2, 2, 1, 2, 3, 0, 2, 0, 2);
  assertCorrectNullPages(columnIndex, true, false, false, true, true, false, true, false, true);
  assertCorrectValues(columnIndex.getMaxValues(), null, -345l, -42l, null, null, 2l, null, 42l, null);
  assertCorrectValues(columnIndex.getMinValues(), null, -532l, -234l, null, null, -42l, null, -3l, null);
  assertCorrectFiltering(columnIndex, eq(col, -42l), 2, 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 1, 2, 3, 4, 6, 8);
  assertCorrectFiltering(columnIndex, notEq(col, -42l), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 2, 5, 7);
  assertCorrectFiltering(columnIndex, gt(col, 2l), 7);
  assertCorrectFiltering(columnIndex, gtEq(col, 2l), 5, 7);
  assertCorrectFiltering(columnIndex, lt(col, -42l), 1, 2);
  assertCorrectFiltering(columnIndex, ltEq(col, -42l), 1, 2, 5);
  assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 1, 2, 5, 7);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
      8);

  builder = ColumnIndexBuilder.getBuilder(type, Integer.MAX_VALUE);
  sb = new StatsBuilder();
  builder.add(sb.stats(type, null, null, null, null, null));
  builder.add(sb.stats(type, 532l, 345l));
  builder.add(sb.stats(type, null, null, null));
  builder.add(sb.stats(type, 234l, 42l, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, 42l, -2l));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, null, null));
  builder.add(sb.stats(type, -3l, -42l));
  assertEquals(9, builder.getPageCount());
  assertEquals(sb.getMinMaxSize(), builder.getMinMaxSize());
  columnIndex = builder.build();
  assertEquals(BoundaryOrder.DESCENDING, columnIndex.getBoundaryOrder());
  assertCorrectNullCounts(columnIndex, 5, 0, 3, 1, 2, 0, 2, 2, 0);
  assertCorrectNullPages(columnIndex, true, false, true, false, true, false, true, true, false);
  assertCorrectValues(columnIndex.getMaxValues(), null, 532l, null, 234l, null, 42l, null, null, -3l);
  assertCorrectValues(columnIndex.getMinValues(), null, 345l, null, 42l, null, -2l, null, null, -42l);
  assertCorrectFiltering(columnIndex, eq(col, 0l), 5);
  assertCorrectFiltering(columnIndex, eq(col, null), 0, 2, 3, 4, 6, 7);
  assertCorrectFiltering(columnIndex, notEq(col, 0l), 0, 1, 2, 3, 4, 5, 6, 7, 8);
  assertCorrectFiltering(columnIndex, notEq(col, null), 1, 3, 5, 8);
  assertCorrectFiltering(columnIndex, gt(col, 2l), 1, 3, 5);
  assertCorrectFiltering(columnIndex, gtEq(col, 2l), 1, 3, 5);
  assertCorrectFiltering(columnIndex, lt(col, -42l));
  assertCorrectFiltering(columnIndex, ltEq(col, -42l), 8);
  assertCorrectFiltering(columnIndex, userDefined(col, LongIsDivisableWith3.class), 1, 3, 5, 8);
  assertCorrectFiltering(columnIndex, invert(userDefined(col, LongIsDivisableWith3.class)), 0, 1, 2, 3, 4, 5, 6, 7,
      8);
}