org.apache.parquet.filter2.compat.FilterCompat.Filter Java Examples

private ParquetReader(Configuration conf,
                      Path file,
                      ReadSupport<T> readSupport,
                      Filter filter) throws IOException {
  this.readSupport = readSupport;
  this.filter = checkNotNull(filter, "filter");
  this.conf = conf;

  FileSystem fs = file.getFileSystem(conf);
  List<FileStatus> statuses = Arrays.asList(fs.listStatus(file, HiddenFileFilter.INSTANCE));
  List<Footer> footers = ParquetFileReader.readAllFootersInParallelUsingSummaryFiles(conf, statuses, false);
  this.footersIterator = footers.iterator();

  for (Footer footer : footers) {
    for(BlockMetaData block : footer.getParquetMetadata().getBlocks()) {
      totalRowCount += block.getRowCount();
    }
  }
}
 

public static List<Group> readFile(File f, Filter filter) throws IOException {
  ParquetReader<Group> reader = createReader(new Path(f.getAbsolutePath()), filter);

  Group current;
  List<Group> users = new ArrayList<Group>();

  current = reader.read();
  while (current != null) {
    users.add(current);
    current = reader.read();
  }

  return users;
}
 

private static ParquetReader<Group> createReader(Path file, Filter filter) throws IOException {
  Configuration conf = new Configuration();
  GroupWriteSupport.setSchema(schema, conf);

  return ParquetReader.builder(new GroupReadSupport(), file)
      .withConf(conf)
      .withFilter(filter)
      .build();
}
 

private List<User> readUsersWithProjection(Filter filter, MessageType schema, boolean useOtherFiltering, boolean useColumnIndexFilter) throws IOException {
  return PhoneBookWriter.readUsers(ParquetReader.builder(new GroupReadSupport(), file)
      .withFilter(filter)
      .useDictionaryFilter(useOtherFiltering)
      .useStatsFilter(useOtherFiltering)
      .useRecordFilter(useOtherFiltering)
      .useColumnIndexFilter(useColumnIndexFilter)
      .set(ReadSupport.PARQUET_READ_SCHEMA, schema.toString()));
}
 

private List<User> readUsers(Filter filter, boolean useOtherFiltering, boolean useColumnIndexFilter)
    throws IOException {
  return PhoneBookWriter.readUsers(ParquetReader.builder(new GroupReadSupport(), file)
      .withFilter(filter)
      .useDictionaryFilter(useOtherFiltering)
      .useStatsFilter(useOtherFiltering)
      .useRecordFilter(useOtherFiltering)
      .useColumnIndexFilter(useColumnIndexFilter));
}
 

private ParquetReader(Configuration conf,
                      Path file,
                      ReadSupport<T> readSupport,
                      FilterCompat.Filter filter) throws IOException {
  this(Collections.singletonList((InputFile) HadoopInputFile.fromPath(file, conf)),
      HadoopReadOptions.builder(conf)
          .withRecordFilter(Objects.requireNonNull(filter, "filter cannot be null"))
          .build(),
      readSupport);
}
 

private void validateFileWithComplexFilter(Path file, List<Group> data) throws IOException {
  Binary binaryValueB = fromString("b");
  Filter filter = FilterCompat.get(
      and(
          gtEq(intColumn("id"), 0),
          and(
              lt(binaryColumn("name"), binaryValueB),
              notEq(binaryColumn("comment"), null))));
  Predicate<Group> predicate = group -> group.getInteger("id", 0) >= 0
      && BINARY_COMPARATOR.compare(group.getBinary("name", 0), binaryValueB) < 0
      && group.getFieldRepetitionCount("comment") > 0;
  validateFile(file, filter, data.stream().filter(predicate));
}
 

private void validateFile(Path file, Filter filter, Stream<Group> data) throws IOException {
  try (ParquetReader<Group> reader = ParquetReader.builder(new GroupReadSupport(), file)
      .withFilter(filter)
      .build()) {
    for (Iterator<Group> it = data.iterator(); it.hasNext();) {
      assertEquals(it.next().toString(), reader.read().toString());
    }
  }
}
 

/**
 * @param readSupport Object which helps reads files of the given type, e.g. Thrift, Avro.
 * @param filter for filtering individual records
 */
public InternalParquetRecordReader(ReadSupport<T> readSupport, Filter filter) {
  this.readSupport = readSupport;
  this.filter = checkNotNull(filter, "filter");
}
 

public Builder<T> withFilter(Filter filter) {
  this.filter = Objects.requireNonNull(filter, "filter cannot be null");
  return this;
}
 

private List<User> readUsers(Filter filter, boolean useOtherFiltering) throws IOException {
  return readUsers(filter, useOtherFiltering, true);
}
 

public static List<BlockMetaData> filterRowGroups(List<FilterLevel> levels, Filter filter, List<BlockMetaData> blocks, ParquetFileReader reader) {
  Objects.requireNonNull(filter, "filter cannot be null");
  return filter.accept(new RowGroupFilter(levels, blocks, reader));
}
 

List<ParquetInputSplit> getSplits(Configuration configuration, List<Footer> footers,
    long maxSplitSize, long minSplitSize, ReadContext readContext)
    throws IOException {
  List<ParquetInputSplit> splits = new ArrayList<ParquetInputSplit>();
  Filter filter = ParquetInputFormat.getFilter(configuration);

  long rowGroupsDropped = 0;
  long totalRowGroups = 0;

  for (Footer footer : footers) {
    final Path file = footer.getFile();
    LOG.debug("{}", file);
    FileSystem fs = file.getFileSystem(configuration);
    FileStatus fileStatus = fs.getFileStatus(file);
    ParquetMetadata parquetMetaData = footer.getParquetMetadata();
    List<BlockMetaData> blocks = parquetMetaData.getBlocks();

    List<BlockMetaData> filteredBlocks;

    totalRowGroups += blocks.size();
    filteredBlocks = RowGroupFilter.filterRowGroups(filter, blocks, parquetMetaData.getFileMetaData().getSchema());
    rowGroupsDropped += blocks.size() - filteredBlocks.size();

    if (filteredBlocks.isEmpty()) {
      continue;
    }

    BlockLocation[] fileBlockLocations = fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
    splits.addAll(
        generateSplits(
            filteredBlocks,
            fileBlockLocations,
            fileStatus,
            readContext.getRequestedSchema().toString(),
            readContext.getReadSupportMetadata(),
            minSplitSize,
            maxSplitSize)
        );
  }

  if (rowGroupsDropped > 0 && totalRowGroups > 0) {
    int percentDropped = (int) ((((double) rowGroupsDropped) / totalRowGroups) * 100);
    LOG.info("Dropping {} row groups that do not pass filter predicate! ({}%)", rowGroupsDropped, percentDropped);
  } else {
    LOG.info("There were no row groups that could be dropped due to filter predicates");
  }
  return splits;
}
 

public Builder<T> withFilter(Filter filter) {
  this.filter = filter;
  optionsBuilder.withRecordFilter(filter);
  return this;
}
 

/**
 * @param readSupport Object which helps reads files of the given type, e.g. Thrift, Avro.
 * @param filter for filtering individual records
 */
public InternalParquetRecordReader(ReadSupport<T> readSupport, Filter filter) {
  this.readSupport = readSupport;
  this.filter = Objects.requireNonNull(filter, "filter cannot be null");
}
 

public <T> RecordReader<T> getRecordReader(final PageReadStore columns,
                                           final RecordMaterializer<T> recordMaterializer,
                                           final Filter filter) {
  Objects.requireNonNull(columns, "columns cannot be null");
  Objects.requireNonNull(recordMaterializer, "recordMaterializer cannot be null");
  Objects.requireNonNull(filter, "filter cannot be null");

  if (leaves.isEmpty()) {
    return new EmptyRecordReader<>(recordMaterializer);
  }

  return filter.accept(new Visitor<RecordReader<T>>() {
    @Override
    public RecordReader<T> visit(FilterPredicateCompat filterPredicateCompat) {

      FilterPredicate predicate = filterPredicateCompat.getFilterPredicate();
      IncrementallyUpdatedFilterPredicateBuilder builder = new IncrementallyUpdatedFilterPredicateBuilder(leaves);
      IncrementallyUpdatedFilterPredicate streamingPredicate = builder.build(predicate);
      RecordMaterializer<T> filteringRecordMaterializer = new FilteringRecordMaterializer<T>(
          recordMaterializer,
          leaves,
          builder.getValueInspectorsByColumn(),
          streamingPredicate);

      return new RecordReaderImplementation<>(
          MessageColumnIO.this,
          filteringRecordMaterializer,
          validating,
          new ColumnReadStoreImpl(columns, filteringRecordMaterializer.getRootConverter(), getType(), createdBy));
    }

    @Override
    public RecordReader<T> visit(UnboundRecordFilterCompat unboundRecordFilterCompat) {
      return new FilteredRecordReader<>(
          MessageColumnIO.this,
          recordMaterializer,
          validating,
          new ColumnReadStoreImpl(columns, recordMaterializer.getRootConverter(), getType(), createdBy),
          unboundRecordFilterCompat.getUnboundRecordFilter(),
          columns.getRowCount()
      );
    }

    @Override
    public RecordReader<T> visit(NoOpFilter noOpFilter) {
      return new RecordReaderImplementation<>(
          MessageColumnIO.this,
          recordMaterializer,
          validating,
          new ColumnReadStoreImpl(columns, recordMaterializer.getRootConverter(), getType(), createdBy));
    }
  });
}
 

public ParquetRecordReader(ReadSupport<T> readSupport, MessageType readSchema, Filter filter) {
	this.filter = checkNotNull(filter, "readSupport");
	this.readSupport = checkNotNull(readSupport, "readSchema");
	this.readSchema = checkNotNull(readSchema, "filter");
}
 

public Builder<T> withFilter(Filter filter) {
  this.filter = checkNotNull(filter, "filter");
  return this;
}
 

public ParquetRecordReader(ReadSupport<T> readSupport, MessageType readSchema, Filter filter) {
	this.filter = checkNotNull(filter, "readSupport");
	this.readSupport = checkNotNull(readSupport, "readSchema");
	this.readSchema = checkNotNull(readSchema, "filter");
}
 

/**
 * @param filter a filter
 * @param blocks a list of block metadata to filter
 * @param schema the file schema
 * @return a filtered list of block metadata
 * @deprecated will be removed in 2.0.0.
 */
@Deprecated
public static List<BlockMetaData> filterRowGroups(Filter filter, List<BlockMetaData> blocks, MessageType schema) {
 Objects.requireNonNull(filter, "filter cannot be null");
  return filter.accept(new RowGroupFilter(blocks, schema));
}
 

/**
 * @param readSupport Object which helps reads files of the given type, e.g. Thrift, Avro.
 * @param filter for filtering individual records
 */
public ParquetRecordReader(ReadSupport<T> readSupport, Filter filter) {
  internalReader = new InternalParquetRecordReader<T>(readSupport, filter);
}
 

/**
 * Returns a non-null Filter, which is a wrapper around either a
 * FilterPredicate, an UnboundRecordFilter, or a no-op filter.
 *
 * @param conf a configuration
 * @return a filter for the unbound record filter specified in conf
 */
public static Filter getFilter(Configuration conf) {
  return FilterCompat.get(getFilterPredicate(conf), getUnboundRecordFilterInstance(conf));
}