Java Code Examples for org.apache.hadoop.hbase.filter.FilterList#addFilter()

The following examples show how to use org.apache.hadoop.hbase.filter.FilterList#addFilter() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ActiveMemberRunner.java From BigDataArchitect with Apache License 2.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList filterList = new FilterList();
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_MEMBER_ID, // 会员id
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION, // 浏览器版本号
            EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME // 添加一个事件名称获取列，在使用singlecolumnvaluefilter的时候必须指定对应的列是一个返回列
    };
    filterList.addFilter(this.getColumnFilter(columns));
    // 只需要page view事件，所以进行过滤
    filterList.addFilter(new SingleColumnValueFilter(ActiveMemberMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.PAGEVIEW.alias)));

    return filterList;
}

Example 2

Source File: DefaultHBaseSerde.java From envelope with Apache License 2.0

6 votes

private FilterList getColumnValueFilters(Row row) {
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
  Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames());
  
  for (Map.Entry<String, ColumnDef> column : columns.entrySet()) {
    if (!column.getValue().cf.equals("rowkey")) {
      if (filterColumnNames.contains(column.getKey())) {
        byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row);
        if (value != null) {
          SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter(
              Bytes.toBytes(column.getValue().cf),
              Bytes.toBytes(column.getValue().name),
              CompareFilter.CompareOp.EQUAL,
              value
          );
          filterList.addFilter(columnValueFilter);
        }
      }
    }
  }
  
  return filterList;
}

Example 3

Source File: RegionLocationCleaner.java From hbase-tools with Apache License 2.0

6 votes

@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}

Example 4

Source File: HBaseReader.java From geowave with Apache License 2.0

6 votes

private void addSkipFilter(final RangeReaderParams<T> params, final FilterList filterList) {
  // Add skipping filter if requested
  if (params.getMaxResolutionSubsamplingPerDimension() != null) {
    if (params.getMaxResolutionSubsamplingPerDimension().length != params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length) {
      LOGGER.warn(
          "Unable to subsample for table '"
              + params.getIndex().getName()
              + "'. Subsample dimensions = "
              + params.getMaxResolutionSubsamplingPerDimension().length
              + " when indexed dimensions = "
              + params.getIndex().getIndexStrategy().getOrderedDimensionDefinitions().length);
    } else {
      final int cardinalityToSubsample =
          IndexUtils.getBitPositionFromSubsamplingArray(
              params.getIndex().getIndexStrategy(),
              params.getMaxResolutionSubsamplingPerDimension());

      final FixedCardinalitySkippingFilter skippingFilter =
          new FixedCardinalitySkippingFilter(cardinalityToSubsample);
      filterList.addFilter(skippingFilter);
    }
  }
}

Example 5

Source File: LocationRunner.java From BigDataPlatform with GNU General Public License v3.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
  //过滤数据，只分析launch事件
  FilterList filterList = new FilterList();
  filterList.addFilter(
      new SingleColumnValueExcludeFilter(Bytes.toBytes(EventLogConstants.EVENT_LOGS_FAMILY_NAME),
          Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL,
          Bytes.toBytes(
              EventEnum.PAGEVIEW.alias)));
  String[] columns = new String[]{
      EventLogConstants.LOG_COLUMN_NAME_SESSION_ID, EventLogConstants.LOG_COLUMN_NAME_UUID,
      EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, EventLogConstants.LOG_COLUMN_NAME_PLATFORM,
      EventLogConstants.LOG_COLUMN_NAME_COUNTRY,
      EventLogConstants.LOG_COLUMN_NAME_PROVINCE,
      EventLogConstants.LOG_COLUMN_NAME_CITY,
      EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME
  };
  filterList.addFilter(this.getColumnFilter(columns));
  return filterList;
}

Example 6

Source File: TestThriftConnection.java From hbase with Apache License 2.0

6 votes

private void testScanWithFilters(Connection connection, String tableName) throws IOException {
  createTable(thriftAdmin, tableName);
  try (Table table = connection.getTable(TableName.valueOf(tableName))){
    FilterList filterList = new FilterList();
    PrefixFilter prefixFilter = new PrefixFilter(Bytes.toBytes("testrow"));
    ColumnValueFilter columnValueFilter = new ColumnValueFilter(FAMILYA, QUALIFIER_1,
        CompareOperator.EQUAL, VALUE_1);
    filterList.addFilter(prefixFilter);
    filterList.addFilter(columnValueFilter);
    Scan scan = new Scan();
    scan.readVersions(2);
    scan.setFilter(filterList);
    ResultScanner scanner = table.getScanner(scan);
    Iterator<Result> iterator = scanner.iterator();
    assertTrue(iterator.hasNext());
    int counter = 0;
    while (iterator.hasNext()) {
      Result result = iterator.next();
      counter += result.size();
    }
    assertEquals(2, counter);
  }
}

Example 7

Source File: HBaseDemo.java From BigDataArchitect with Apache License 2.0

6 votes

/**
 * 查询某个用户所有的主叫电话（type=1）
 *  某个用户
 *  type=1
 *
 */
@Test
public void getType() throws IOException {
    Scan scan = new Scan();
    //创建过滤器集合
    FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    //创建过滤器
    SingleColumnValueFilter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"),Bytes.toBytes("type"),CompareOperator.EQUAL,Bytes.toBytes("1"));
    filters.addFilter(filter1);
    //前缀过滤器
    PrefixFilter filter2 = new PrefixFilter(Bytes.toBytes("15883348450"));
    filters.addFilter(filter2);
    scan.setFilter(filters);
    ResultScanner rss = table.getScanner(scan);
    for (Result rs:rss) {
        System.out.print(Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("dnum")))));
        System.out.print("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("type")))));
        System.out.print("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("length")))));
        System.out.println("--"+Bytes.toString(CellUtil.cloneValue(rs.getColumnLatestCell(Bytes.toBytes("cf"),Bytes.toBytes("date")))));
    }
}

Example 8

Source File: VertexIndexModel.java From hgraphdb with Apache License 2.0

6 votes

private Scan getVertexIndexScanWithLimit(String label, boolean isUnique, String key, Object from, int limit, boolean reversed) {
    byte[] prefix = serializeForRead(label, isUnique, key, null);
    byte[] startRow = from != null
            ? serializeForRead(label, isUnique, key, from)
            : prefix;
    byte[] stopRow = HConstants.EMPTY_END_ROW;
    if (graph.configuration().getInstanceType() == HBaseGraphConfiguration.InstanceType.BIGTABLE) {
        if (reversed) {
            throw new UnsupportedOperationException("Reverse scans not supported by Bigtable");
        } else {
            // PrefixFilter in Bigtable does not automatically stop
            // See https://github.com/GoogleCloudPlatform/cloud-bigtable-client/issues/1087
            stopRow = HBaseGraphUtils.incrementBytes(prefix);
        }
    }
    if (reversed) startRow = HBaseGraphUtils.incrementBytes(startRow);
    Scan scan = new Scan(startRow, stopRow);
    FilterList filterList = new FilterList();
    filterList.addFilter(new PrefixFilter(prefix));
    filterList.addFilter(new PageFilter(limit));
    scan.setFilter(filterList);
    scan.setReversed(reversed);
    return scan;
}

Example 9

Source File: CommandAdapter.java From hbase-tools with Apache License 2.0

6 votes

public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}

Example 10

Source File: JobHistoryService.java From hraven with Apache License 2.0

6 votes

/**
 * creates a scan for flow data
 * @param rowPrefix - start row prefix
 * @param limit - limit on scanned results
 * @param version - version to match
 * @return Scan
 */
private Scan createFlowScan(byte[] rowPrefix, int limit, String version) {
  Scan scan = new Scan();
  scan.setStartRow(rowPrefix);

  // using a large scanner caching value with a small limit can mean we scan a
  // lot more data than necessary, so lower the caching for low limits
  scan.setCaching(Math.min(limit, defaultScannerCaching));
  // require that all rows match the prefix we're looking for
  Filter prefixFilter = new WhileMatchFilter(new PrefixFilter(rowPrefix));
  // if version is passed, restrict the rows returned to that version
  if (version != null && version.length() > 0) {
    FilterList filters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    filters.addFilter(prefixFilter);
    filters.addFilter(new SingleColumnValueFilter(Constants.INFO_FAM_BYTES,
        Constants.VERSION_COLUMN_BYTES, CompareFilter.CompareOp.EQUAL,
        Bytes.toBytes(version)));
    scan.setFilter(filters);
  } else {
    scan.setFilter(prefixFilter);
  }
  return scan;
}

Example 11

Source File: PageViewRunner.java From BigDataArchitect with Apache License 2.0

6 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList filterList = new FilterList();
    // 只需要pageview事件
    filterList.addFilter(new SingleColumnValueFilter(PageViewMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventLogConstants.EventEnum.PAGEVIEW.alias)));
    // 定义mapper中需要获取的列名
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME, // 获取事件名称
            EventLogConstants.LOG_COLUMN_NAME_CURRENT_URL, // 当前url
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
            EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本号
    };
    filterList.addFilter(this.getColumnFilter(columns));

    return filterList;
}

Example 12

Source File: HGet.java From spliceengine with GNU Affero General Public License v3.0

6 votes

@Override
public void setFilter(DataFilter txnFilter){
    Filter toAdd;
    Filter existingFilter=get.getFilter();
    if(existingFilter!=null){
        FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        fl.addFilter(existingFilter);
        if(txnFilter instanceof HFilterWrapper)
            fl.addFilter(((HFilterWrapper)txnFilter).unwrapDelegate());
        else
            fl.addFilter(new HDataFilterWrapper(txnFilter));
        toAdd = fl;
    }else{
        if(txnFilter instanceof HFilterWrapper)
            toAdd = ((HFilterWrapper)txnFilter).unwrapDelegate();
        else
            toAdd = new HDataFilterWrapper(txnFilter);
    }
    get.setFilter(toAdd);
}

Example 13

Source File: HbaseTraceDaoV2.java From pinpoint with Apache License 2.0

5 votes

private Filter getFilter(Filter filter1, Filter filter2) {
    if (filter1 != null && filter2 != null) {
        FilterList filterList = new FilterList();
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);
        return filterList;
    }
    if (filter1 != null) {
        return filter1;
    }
    if (filter2 != null) {
        return filter2;
    }
    return null;
}

Example 14

Source File: Filters.java From java-docs-samples with Apache License 2.0

5 votes

public static void filterComposingInterleave(
    String projectId, String instanceId, String tableId) {
  // A filter that matches cells with the value true OR with the column qualifier os_build
  Filter qualifierFilter =
      new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("os_build")));
  Filter valueFilter =
      new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("true")));

  FilterList filter = new FilterList(Operator.MUST_PASS_ONE);
  filter.addFilter(qualifierFilter);
  filter.addFilter(valueFilter);

  Scan scan = new Scan().setFilter(filter).setMaxVersions();
  readWithFilter(projectId, instanceId, tableId, scan);
}

Example 15

Source File: HBaseResourceStore.java From kylin-on-parquet-v2 with Apache License 2.0

5 votes

private FilterList generateTimeFilterList(VisitFilter visitFilter) {
    FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
    if (visitFilter.lastModStart >= 0) { // NOTE: Negative value does not work in its binary form
        SingleColumnValueFilter timeStartFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS,
                CompareFilter.CompareOp.GREATER_OR_EQUAL, Bytes.toBytes(visitFilter.lastModStart));
        filterList.addFilter(timeStartFilter);
    }
    if (visitFilter.lastModEndExclusive != Long.MAX_VALUE) {
        SingleColumnValueFilter timeEndFilter = new SingleColumnValueFilter(B_FAMILY, B_COLUMN_TS,
                CompareFilter.CompareOp.LESS, Bytes.toBytes(visitFilter.lastModEndExclusive));
        filterList.addFilter(timeEndFilter);
    }
    return filterList.getFilters().isEmpty() ? null : filterList;
}

Example 16

Source File: HBaseLogReader.java From eagle with Apache License 2.0

5 votes

/**
 * TODO If the required field is null for a row, then this row will not be fetched. That could be a
 * problem for counting Need another version of read to strictly get the number of rows which will return
 * all the columns for a column family
 */
@Override
public void open() throws IOException {
    if (isOpen) {
        return; // silently return
    }
    try {
        tbl = EagleConfigFactory.load().getHTable(schema.getTable());
    } catch (RuntimeException ex) {
        throw new IOException(ex);
    }

    String rowkeyRegex = buildRegex2(searchTags);
    RegexStringComparator regexStringComparator = new RegexStringComparator(rowkeyRegex);
    regexStringComparator.setCharset(Charset.forName("ISO-8859-1"));
    RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator);
    FilterList filterList = new FilterList();
    filterList.addFilter(filter);
    Scan s1 = new Scan();
    // reverse timestamp, startRow is stopKey, and stopRow is startKey
    s1.setStartRow(stopKey);
    s1.setStopRow(startKey);
    s1.setFilter(filterList);
    // TODO the # of cached rows should be minimum of (pagesize and 100)
    s1.setCaching(100);
    // TODO not optimized for all applications
    s1.setCacheBlocks(true);
    // scan specified columnfamily and qualifiers
    for (byte[] qualifier : qualifiers) {
        s1.addColumn(schema.getColumnFamily().getBytes(), qualifier);
    }
    rs = tbl.getScanner(s1);
    isOpen = true;
}

Example 17

Source File: InboundRunner.java From BigDataArchitect with Apache License 2.0

5 votes

@Override
protected Filter fetchHbaseFilter() {
    FilterList list = new FilterList();
    String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_REFERRER_URL, // 前一个页面的url
            EventLogConstants.LOG_COLUMN_NAME_UUID, // uuid
            EventLogConstants.LOG_COLUMN_NAME_SESSION_ID, // 会话id
            EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
            EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
            EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME // 事件名称
    };
    list.addFilter(this.getColumnFilter(columns));
    list.addFilter(new SingleColumnValueFilter(InboundMapper.family, Bytes.toBytes(EventLogConstants.LOG_COLUMN_NAME_EVENT_NAME), CompareOp.EQUAL, Bytes.toBytes(EventEnum.PAGEVIEW.alias)));
    return list;
}

Example 18

Source File: NewMemberRunner.java From BigDataPlatform with GNU General Public License v3.0

5 votes

@Override
protected Filter fetchHbaseFilter() {
  FilterList filterList = new FilterList();
  // 定义mapper中需要获取的列名
  String[] columns = new String[] { EventLogConstants.LOG_COLUMN_NAME_MEMBER_ID, // 会员id
      EventLogConstants.LOG_COLUMN_NAME_SERVER_TIME, // 服务器时间
      EventLogConstants.LOG_COLUMN_NAME_PLATFORM, // 平台名称
      EventLogConstants.LOG_COLUMN_NAME_BROWSER_NAME, // 浏览器名称
      EventLogConstants.LOG_COLUMN_NAME_BROWSER_VERSION // 浏览器版本信息
  };
  filterList.addFilter(this.getColumnFilter(columns));
  return filterList;
}

Example 19

Source File: HBaseLogReader.java From Eagle with Apache License 2.0

5 votes

/**
 * TODO If the required field is null for a row, then this row will not be fetched. That could be a problem for counting
 * Need another version of read to strictly get the number of rows which will return all the columns for a column family
 */
public void open() throws IOException {
	if (isOpen)
		return; // silently return
	try {
		tbl = EagleConfigFactory.load().getHTable(schema.getTable());
	} catch (RuntimeException ex) {
		throw new IOException(ex);
	}

	String rowkeyRegex = buildRegex2(searchTags);
	RegexStringComparator regexStringComparator = new RegexStringComparator(
			rowkeyRegex);
	regexStringComparator.setCharset(Charset.forName("ISO-8859-1"));
	RowFilter filter = new RowFilter(CompareOp.EQUAL, regexStringComparator);
	FilterList filterList = new FilterList();
	filterList.addFilter(filter);
	Scan s1 = new Scan();
	// reverse timestamp, startRow is stopKey, and stopRow is startKey
	s1.setStartRow(stopKey);
	s1.setStopRow(startKey);
	s1.setFilter(filterList);
	// TODO the # of cached rows should be minimum of (pagesize and 100)
	s1.setCaching(100);
	// TODO not optimized for all applications
	s1.setCacheBlocks(true);
	// scan specified columnfamily and qualifiers
	for(byte[] qualifier : qualifiers){
		s1.addColumn(schema.getColumnFamily().getBytes(), qualifier);
	}
	rs = tbl.getScanner(s1);
	isOpen = true;
}

Example 20

Source File: QuotaTableUtil.java From hbase with Apache License 2.0

4 votes

/**
 * converts quotafilter to serializeable filterlists.
 */
public static Filter makeFilter(final QuotaFilter filter) {
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  if (StringUtils.isNotEmpty(filter.getUserFilter())) {
    FilterList userFilters = new FilterList(FilterList.Operator.MUST_PASS_ONE);
    boolean hasFilter = false;

    if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) {
      FilterList nsFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
      nsFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
      nsFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL,
          new RegexStringComparator(
            getSettingsQualifierRegexForUserNamespace(filter.getNamespaceFilter()), 0)));
      userFilters.addFilter(nsFilters);
      hasFilter = true;
    }
    if (StringUtils.isNotEmpty(filter.getTableFilter())) {
      FilterList tableFilters = new FilterList(FilterList.Operator.MUST_PASS_ALL);
      tableFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
      tableFilters.addFilter(new QualifierFilter(CompareOperator.EQUAL,
          new RegexStringComparator(
            getSettingsQualifierRegexForUserTable(filter.getTableFilter()), 0)));
      userFilters.addFilter(tableFilters);
      hasFilter = true;
    }
    if (!hasFilter) {
      userFilters.addFilter(new RowFilter(CompareOperator.EQUAL,
          new RegexStringComparator(getUserRowKeyRegex(filter.getUserFilter()), 0)));
    }

    filterList.addFilter(userFilters);
  } else if (StringUtils.isNotEmpty(filter.getTableFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL,
        new RegexStringComparator(getTableRowKeyRegex(filter.getTableFilter()), 0)));
  } else if (StringUtils.isNotEmpty(filter.getNamespaceFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL,
        new RegexStringComparator(getNamespaceRowKeyRegex(filter.getNamespaceFilter()), 0)));
  } else if (StringUtils.isNotEmpty(filter.getRegionServerFilter())) {
    filterList.addFilter(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(
        getRegionServerRowKeyRegex(filter.getRegionServerFilter()), 0)));
  }
  return filterList;
}