Java Code Examples for org.apache.hadoop.hbase.filter.FilterList.Operator

The following examples show how to use org.apache.hadoop.hbase.filter.FilterList.Operator. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: envelope   Source File: DefaultHBaseSerde.java    License: Apache License 2.0 6 votes vote down vote up
private FilterList getColumnValueFilters(Row row) {
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
  Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames());
  
  for (Map.Entry<String, ColumnDef> column : columns.entrySet()) {
    if (!column.getValue().cf.equals("rowkey")) {
      if (filterColumnNames.contains(column.getKey())) {
        byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row);
        if (value != null) {
          SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter(
              Bytes.toBytes(column.getValue().cf),
              Bytes.toBytes(column.getValue().name),
              CompareFilter.CompareOp.EQUAL,
              value
          );
          filterList.addFilter(columnValueFilter);
        }
      }
    }
  }
  
  return filterList;
}
 
Example 2
Source Project: java-docs-samples   Source File: Filters.java    License: Apache License 2.0 6 votes vote down vote up
public static void filterLimitValueRange(String projectId, String instanceId, String tableId) {
  // A filter that matches cells whose values are between the given values
  ValueFilter valueGreaterFilter =
      new ValueFilter(
          CompareFilter.CompareOp.GREATER_OR_EQUAL,
          new BinaryComparator(Bytes.toBytes("PQ2A.190405")));
  ValueFilter valueLesserFilter =
      new ValueFilter(
          CompareFilter.CompareOp.LESS_OR_EQUAL,
          new BinaryComparator(Bytes.toBytes("PQ2A.190406")));

  FilterList filter = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  filter.addFilter(valueGreaterFilter);
  filter.addFilter(valueLesserFilter);

  Scan scan = new Scan().setFilter(filter);
  readWithFilter(projectId, instanceId, tableId, scan);
}
 
Example 3
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters
 * we expect to get the same result as the inclusive stop result.
 * @throws Exception
 */
@Test
public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception {
  byte[] r1 = Bytes.toBytes("Row1");
  byte[] r11 = Bytes.toBytes("Row11");
  byte[] r2 = Bytes.toBytes("Row2");

  FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  flist.addFilter(new AlwaysNextColFilter());
  flist.addFilter(new InclusiveStopFilter(r1));
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));

  flist.reset();
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
  assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2)));
}
 
Example 4
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test serialization
 * @throws Exception
 */
@Test
public void testSerialization() throws Exception {
  List<Filter> filters = new ArrayList<>();
  filters.add(new PageFilter(MAX_PAGES));
  filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
  Filter filterMPALL =
    new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);

  // Decompose filterMPALL to bytes.
  byte[] buffer = filterMPALL.toByteArray();

  // Recompose filterMPALL.
  FilterList newFilter = FilterList.parseFrom(buffer);

  // Run tests
  mpOneTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getFilterMPONE())));
  mpAllTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getMPALLFilter())));
  orderingTest(ProtobufUtil.toFilter(ProtobufUtil.toFilter(getOrderingFilter())));
}
 
Example 5
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWithMultiVersionsInSameRow() throws Exception {
  FilterList filterList01 =
      new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0));

  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
      1, Bytes.toBytes("value"));
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
      2, Bytes.toBytes("value"));
  KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
      3, Bytes.toBytes("value"));

  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterCell(kv1));
  assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv2));
  assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv3));

  FilterList filterList11 =
      new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));

  assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv1));
  assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv2));
  assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv3));
}
 
Example 6
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMPONEWithSeekNextUsingHint() throws Exception {
  byte[] col = Bytes.toBytes("c");
  FilterList filterList =
      new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col));

  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2,
      Bytes.toBytes("value"));
  KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3,
      Bytes.toBytes("value"));
  KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
      Bytes.toBytes("value"));

  assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
  assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv2));
  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3));
  assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv4));
}
 
Example 7
Source Project: java-docs-samples   Source File: Filters.java    License: Apache License 2.0 5 votes vote down vote up
public static void filterComposingChain(String projectId, String instanceId, String tableId) {
  // A filter that selects one cell per row AND within the column family cell_plan
  Filter familyFilter =
      new FamilyFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("cell_plan")));
  Filter columnCountGetFilter = new ColumnCountGetFilter(3);

  FilterList filter = new FilterList(FilterList.Operator.MUST_PASS_ALL);
  filter.addFilter(columnCountGetFilter);
  filter.addFilter(familyFilter);
  Scan scan = new Scan().setFilter(filter);
  readWithFilter(projectId, instanceId, tableId, scan);
}
 
Example 8
Source Project: java-docs-samples   Source File: Filters.java    License: Apache License 2.0 5 votes vote down vote up
public static void filterComposingInterleave(
    String projectId, String instanceId, String tableId) {
  // A filter that matches cells with the value true OR with the column qualifier os_build
  Filter qualifierFilter =
      new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("os_build")));
  Filter valueFilter =
      new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("true")));

  FilterList filter = new FilterList(Operator.MUST_PASS_ONE);
  filter.addFilter(qualifierFilter);
  filter.addFilter(valueFilter);

  Scan scan = new Scan().setFilter(filter).setMaxVersions();
  readWithFilter(projectId, instanceId, tableId, scan);
}
 
Example 9
Source Project: hbase   Source File: TestFuzzyRowFilterEndToEnd.java    License: Apache License 2.0 5 votes vote down vote up
private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
    throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes(cf));
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 
Example 10
Source Project: hbase   Source File: TestFilterListOnMini.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFiltersWithOR() throws Exception {
  TableName tn = TableName.valueOf(name.getMethodName());
  Table table = TEST_UTIL.createTable(tn, new String[] { "cf1", "cf2" });
  byte[] CF1 = Bytes.toBytes("cf1");
  byte[] CF2 = Bytes.toBytes("cf2");
  Put put1 = new Put(Bytes.toBytes("0"));
  put1.addColumn(CF1, Bytes.toBytes("col_a"), Bytes.toBytes(0));
  table.put(put1);
  Put put2 = new Put(Bytes.toBytes("0"));
  put2.addColumn(CF2, Bytes.toBytes("col_b"), Bytes.toBytes(0));
  table.put(put2);
  FamilyFilter filterCF1 =
      new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF1));
  FamilyFilter filterCF2 =
      new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(CF2));
  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  filterList.addFilter(filterCF1);
  filterList.addFilter(filterCF2);
  Scan scan = new Scan();
  scan.setFilter(filterList);
  ResultScanner scanner = table.getScanner(scan);
  LOG.info("Filter list: " + filterList);
  for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    Assert.assertEquals(2, rr.size());
  }
}
 
Example 11
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAddFilter() throws Exception {
  Filter filter1 = new FirstKeyOnlyFilter();
  Filter filter2 = new FirstKeyOnlyFilter();

  FilterList filterList = new FilterList(filter1, filter2);
  filterList.addFilter(new FirstKeyOnlyFilter());

  filterList = new FilterList(Arrays.asList(filter1, filter2));
  filterList.addFilter(new FirstKeyOnlyFilter());

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  filterList.addFilter(new FirstKeyOnlyFilter());

  filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
  filterList.addFilter(new FirstKeyOnlyFilter());

  filterList.setReversed(false);
  FirstKeyOnlyFilter f = new FirstKeyOnlyFilter();
  f.setReversed(true);
  try {
    filterList.addFilter(f);
    fail("The IllegalArgumentException should be thrown because the added filter is reversed");
  } catch (IllegalArgumentException e) {
  }

}
 
Example 12
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
private Filter getFilterMPONE() {
  List<Filter> filters = new ArrayList<>();
  filters.add(new PageFilter(MAX_PAGES));
  filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
  Filter filterMPONE =
    new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
  return filterMPONE;
}
 
Example 13
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
private Filter getMPALLFilter() {
  List<Filter> filters = new ArrayList<>();
  filters.add(new PageFilter(MAX_PAGES));
  filters.add(new WhileMatchFilter(new PrefixFilter(Bytes.toBytes("yyy"))));
  Filter filterMPALL =
    new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
  return filterMPALL;
}
 
Example 14
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
public Filter getOrderingFilter() {
  List<Filter> filters = new ArrayList<>();
  filters.add(new PrefixFilter(Bytes.toBytes("yyy")));
  filters.add(new PageFilter(MAX_PAGES));
  Filter filterMPONE =
    new FilterList(FilterList.Operator.MUST_PASS_ONE, filters);
  return filterMPONE;
}
 
Example 15
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * When we do a "MUST_PASS_ONE" (a logical 'OR') of the above two filters
 * we expect to get the same result as the 'prefix' only result.
 * @throws Exception
 */
@Test
public void testFilterListTwoFiltersMustPassOne() throws Exception {
  byte[] r1 = Bytes.toBytes("Row1");
  byte[] r11 = Bytes.toBytes("Row11");
  byte[] r2 = Bytes.toBytes("Row2");

  FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  flist.addFilter(new PrefixFilter(r1));
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));

  flist.reset();
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
  assertEquals(ReturnCode.SKIP, flist.filterCell(new KeyValue(r2, r2, r2)));

  flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  flist.addFilter(new AlwaysNextColFilter());
  flist.addFilter(new PrefixFilter(r1));
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));

  flist.reset();
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
  assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2)));
}
 
Example 16
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests the behavior of transform() in a hierarchical filter.
 *
 * transform() only applies after a filterCell() whose return-code includes the KeyValue.
 * Lazy evaluation of AND
 */
@Test
public void testTransformMPO() throws Exception {
  // Apply the following filter:
  //     (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
  //  OR (family=fam AND qualifier=qual2)
  final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
          new KeyOnlyFilter())),
      new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
          new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
          new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));

  final KeyValue kvQual1 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
  final KeyValue kvQual2 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
  final KeyValue kvQual3 = new KeyValue(
      Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));

  // Value for fam:qual1 should be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual1));
  final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
  assertEquals(0, transformedQual1.getValueLength());

  // Value for fam:qual2 should not be stripped:
  assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual2));
  final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
  assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
    transformedQual2.getValueOffset(), transformedQual2.getValueLength()));

  // Other keys should be skipped:
  assertEquals(Filter.ReturnCode.SKIP, flist.filterCell(kvQual3));
}
 
Example 17
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTheMaximalRule() throws IOException {
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
  MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
  MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
  MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
  MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
  MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
  MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW);

  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3);
  assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6);
  assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6);
  assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1);
  assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5);
  assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter2,
      new FilterList(Operator.MUST_PASS_ALL, filter3, filter4));
  assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7);
  assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
}
 
Example 18
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTheMinimalRule() throws IOException {
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
  MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
  MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
  MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
  MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
  MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
  FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6);
  assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6);
  assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1);
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5);
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
      new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
      new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
  assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
  assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
}
 
Example 19
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  Filter filter1 = new MockSeekHintFilter(kv1);
  filter1.setReversed(true);
  Filter filter2 = new MockSeekHintFilter(kv2);
  filter2.setReversed(true);
  Filter filter3 = new MockSeekHintFilter(kv3);
  filter3.setReversed(true);

  FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
  filterList.setReversed(true);
  filterList.addFilter(filter1);
  filterList.addFilter(filter2);
  filterList.addFilter(filter3);

  Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
  Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));

  filterList = new FilterList(Operator.MUST_PASS_ALL);
  filterList.setReversed(true);
  filterList.addFilter(filter1);
  filterList.addFilter(filter2);
  filterList.addFilter(filter3);

  Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
  Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
}
 
Example 20
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testReversedFilterListWithOR() throws IOException {
  byte[] r22 = Bytes.toBytes("Row22");
  byte[] r2 = Bytes.toBytes("Row2");
  byte[] r1 = Bytes.toBytes("Row1");

  FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  filterList.setReversed(true);
  PrefixFilter prefixFilter = new PrefixFilter(r2);
  prefixFilter.setReversed(true);
  filterList.addFilter(prefixFilter);
  filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r22, r22, r22)));
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r2, r2, r2)));

  filterList.reset();
  filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.SKIP, filterList.filterCell(new KeyValue(r1, r1, r1)));

  filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  filterList.setReversed(true);
  AlwaysNextColFilter alwaysNextColFilter = new AlwaysNextColFilter();
  alwaysNextColFilter.setReversed(true);
  prefixFilter = new PrefixFilter(r2);
  prefixFilter.setReversed(true);
  filterList.addFilter(alwaysNextColFilter);
  filterList.addFilter(prefixFilter);
  filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r22));
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r22, r22, r22)));
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(new KeyValue(r2, r2, r2)));

  filterList.reset();
  filterList.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(new KeyValue(r1, r1, r1)));
}
 
Example 21
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyOnlyFilterTransformCell() throws IOException {
  Cell c;
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
      1, Bytes.toBytes("value1"));
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
      2, Bytes.toBytes("value2"));

  Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
      CompareOperator.EQUAL, Bytes.toBytes("value1"));
  Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
      CompareOperator.EQUAL, Bytes.toBytes("value2"));
  FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);

  FilterList keyOnlyFilterFirst =
      new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);

  assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv1));
  c = keyOnlyFilterFirst.transformCell(kv1);
  assertEquals(0, c.getValueLength());
  assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv2));
  c = keyOnlyFilterFirst.transformCell(kv2);
  assertEquals(0, c.getValueLength());

  internalFilterList.reset();
  FilterList keyOnlyFilterLast =
      new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
  assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv1));
  c = keyOnlyFilterLast.transformCell(kv1);
  assertEquals(0, c.getValueLength());
  assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv2));
  c = keyOnlyFilterLast.transformCell(kv2);
  assertEquals(0, c.getValueLength());
}
 
Example 22
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyFilterListTransformCell() throws IOException {
  KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
      1, Bytes.toBytes("value"));
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
  assertEquals(kv, filterList.transformCell(kv));

  filterList = new FilterList(Operator.MUST_PASS_ONE);
  assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
  assertEquals(kv, filterList.transformCell(kv));
}
 
Example 23
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRowCountFilter() throws IOException {
  KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1,
      Bytes.toBytes("value"));
  KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2,
      Bytes.toBytes("value"));
  MockNextRowFilter mockNextRowFilter = new MockNextRowFilter();
  FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter);
  filter.filterCell(kv1);
  filter.filterCell(kv2);
  assertEquals(2, mockNextRowFilter.getHitCount());
}
 
Example 24
Source Project: hbase   Source File: TestFilterList.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTransformCell() throws IOException {
  KeyValue kv =
      new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), 1,
          Bytes.toBytes("value"));

  // case MUST_PASS_ONE
  TransformFilter filter1 = new TransformFilter(ReturnCode.INCLUDE);
  TransformFilter filter2 = new TransformFilter(ReturnCode.NEXT_ROW);
  TransformFilter filter3 = new TransformFilter(ReturnCode.SEEK_NEXT_USING_HINT);
  FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2, filter3);
  Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
  Assert.assertEquals(kv, filterList.transformCell(kv));
  Assert.assertEquals(true, filter1.getTransformed());
  Assert.assertEquals(false, filter2.getTransformed());
  Assert.assertEquals(false, filter3.getTransformed());

  // case MUST_PASS_ALL
  filter1 = new TransformFilter(ReturnCode.INCLUDE);
  filter2 = new TransformFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
  filter3 = new TransformFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
  filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2, filter3);

  Assert.assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv));
  Assert.assertEquals(kv, filterList.transformCell(kv));
  Assert.assertEquals(true, filter1.getTransformed());
  Assert.assertEquals(true, filter2.getTransformed());
  Assert.assertEquals(true, filter3.getTransformed());
}
 
Example 25
Source Project: phoenix   Source File: WhereOptimizerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testForceSkipScanOnSaltedTable() throws SQLException {
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement().execute("CREATE TABLE IF NOT EXISTS user_messages (\n" + 
            "        SENDER_ID UNSIGNED_LONG NOT NULL,\n" + 
            "        RECIPIENT_ID UNSIGNED_LONG NOT NULL,\n" + 
            "        SENDER_IP VARCHAR,\n" + 
            "        IS_READ VARCHAR,\n" + 
            "        IS_DELETED VARCHAR,\n" + 
            "        M_TEXT VARCHAR,\n" + 
            "        M_TIMESTAMP timestamp  NOT NULL,\n" + 
            "        ROW_ID UNSIGNED_LONG NOT NULL\n" + 
            "        constraint rowkey primary key (SENDER_ID,RECIPIENT_ID,M_TIMESTAMP DESC,ROW_ID))\n" + 
            "SALT_BUCKETS=12\n");
    String query = "select /*+ SKIP_SCAN */ count(*) from user_messages where is_read='N' and recipient_id=5399179882";
    StatementContext context = compileStatement(query);
    Scan scan = context.getScan();
    Filter filter = scan.getFilter();
    
    assertNotNull(filter);
    assertTrue(filter instanceof FilterList);
    FilterList filterList = (FilterList)filter;
    assertEquals(FilterList.Operator.MUST_PASS_ALL, filterList.getOperator());
    assertEquals(2, filterList.getFilters().size());
    assertTrue(filterList.getFilters().get(0) instanceof SkipScanFilter);
    assertTrue(filterList.getFilters().get(1) instanceof SingleKeyValueComparisonFilter);
    
    ScanRanges scanRanges = context.getScanRanges();
    assertNotNull(scanRanges);
    assertEquals(3,scanRanges.getRanges().size());
    assertEquals(1,scanRanges.getRanges().get(1).size());
    assertEquals(KeyRange.EVERYTHING_RANGE,scanRanges.getRanges().get(1).get(0));
    assertEquals(1,scanRanges.getRanges().get(2).size());
    assertTrue(scanRanges.getRanges().get(2).get(0).isSingleKey());
    assertEquals(Long.valueOf(5399179882L), PUnsignedLong.INSTANCE.toObject(scanRanges.getRanges().get(2).get(0).getLowerRange()));
}
 
Example 26
Source Project: pxf   Source File: HBaseUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * get StringBuilder with print according to given {@link FilterList}
 * 
 * @param result as {@link StringBuilder}
 * @param filterList {@link FilterList} with HBase filters
 */
public static void getFilterListPrint(StringBuilder result, FilterList filterList) {

	if (filterList == null || filterList.getFilters().size() == 0) {
		return;
	}

	if (result == null) {
		result = new StringBuilder();
	}

	List<Filter> list = filterList.getFilters();

	Operator operator = filterList.getOperator();
	String operatorString = (operator.equals(Operator.MUST_PASS_ALL)) ? " AND " : " OR ";

	// open current filter list
	result.append("[");

	int listIndex = 0;

	// go over each filter in the list
	for (Filter filter : list) {

		// if current filter is a FilterList, call the method again with it
		if (filter instanceof FilterList) {
			getFilterListPrint(result, (FilterList) filter);
		} else {

			// append filter to result
			result.append("(" + filter + ")");

			// if end of list and not FilterList, close the current list
			if (listIndex == list.size() - 1) {
				result.append("]");
			}
		}

		// if still not the end of list put the operator
		if (listIndex != list.size() - 1) {
			result.append(operatorString);
		}

		listIndex++;
	}
}
 
Example 27
Source Project: Eagle   Source File: HBaseFilterBuilder.java    License: Apache License 2.0 4 votes vote down vote up
/**
	 * @see org.apache.eagle.query.parser.TokenType
	 *
	 * @return
	 */
	public FilterList buildFilters(){
		// TODO: Optimize to select between row filter or column filter for better performance
		// Use row key filter priority by default
		boolean rowFilterPriority = true;

		FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
		for(ANDExpression andExpr : _orExpr.getANDExprList()){
			
			FilterList list = new FilterList(Operator.MUST_PASS_ALL);
			Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
			List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
//			List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();

			// TODO refactor not to use too much if/else
			for(AtomicExpression ae : andExpr.getAtomicExprList()){
				// TODO temporarily ignore those fields which are not for attributes

				String fieldName = ae.getKey();
				if(ae.getKeyType() == TokenType.ID){
					fieldName = parseEntityAttribute(fieldName);
					if(fieldName == null){
						LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
						continue;
					}
				}

				String value = ae.getValue();
				ComparisonOperator op = ae.getOp();
				TokenType keyType = ae.getKeyType();
				TokenType valueType = ae.getValueType();
				QualifierFilterEntity entry = new QualifierFilterEntity(fieldName,value,op,keyType,valueType);

				// TODO Exact match, need to add escape for those special characters here, including:
				// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"

				if(keyType == TokenType.ID && isTag(fieldName)){
					if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
							&& !TokenType.NULL.equals(valueType))
					{
						// Use RowFilter for equal TAG
						if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
						tagFilters.get(fieldName).add(value);
					} else if (rowFilterPriority && ComparisonOperator.IN.equals(op))
					{
						// Use RowFilter here by default
						if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
						tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
					} else if (ComparisonOperator.LIKE.equals(op)
						|| ComparisonOperator.NOT_LIKE.equals(op)
						|| ComparisonOperator.CONTAINS.equals(op)
						|| ComparisonOperator.NOT_CONTAINS.equals(op)
						|| ComparisonOperator.IN.equals(op)
						|| ComparisonOperator.IS.equals(op)
						|| ComparisonOperator.IS_NOT.equals(op)
						|| ComparisonOperator.NOT_EQUAL.equals(op)
						|| ComparisonOperator.EQUAL.equals(op)
						|| ComparisonOperator.NOT_IN.equals(op))
					{
						qualifierFilters.add(entry);
					} else
					{
						LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName + " yet, going to ignore");
						throw new IllegalArgumentException("Don't support operation: "+op+" on tag field: "+fieldName+", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
					}
				}else{
					qualifierFilters.add(entry);
				}
			}

			// Build RowFilter for equal tags
			list.addFilter(buildTagFilter(tagFilters));

			// Build SingleColumnValueFilter
			FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
			if(qualifierFilterList != null && qualifierFilterList.getFilters().size()>0){
				list.addFilter(qualifierFilterList);
			}else {
				if(LOG.isDebugEnabled()) LOG.debug("Ignore empty qualifier filter from "+qualifierFilters.toString());
			}
			fltList.addFilter(list);
		}
		LOG.info("Query: " + _orExpr.toString() + " => Filter: " + fltList.toString());
		return fltList;
	}
 
Example 28
Source Project: Eagle   Source File: HBaseFilterBuilder.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * all qualifiers' condition must be satisfied.
 *
 * <H1>Use RegexStringComparator for:</H1>
 *      IN
 *      LIKE
 *      NOT_LIKE
 *
 * <H1>Use SubstringComparator for:</H1>
 *      CONTAINS
 *
 * <H1>Use EntityQualifierHelper for:</H1>
 *      EQUALS
 *      NOT_EUQALS
 *      LESS
 *      LESS_OR_EQUAL
 *      GREATER
 *      GREATER_OR_EQUAL
 *
 * <H2>
 *     TODO: Compare performance of RegexStringComparator ,SubstringComparator ,EntityQualifierHelper
 * </H2>
 *
 * @param qualifierFilters
 * @return
 */
protected FilterList buildQualifierFilter(List<QualifierFilterEntity> qualifierFilters){
	FilterList list = new FilterList(Operator.MUST_PASS_ALL);
	// iterate all the qualifiers
	for(QualifierFilterEntity entry : qualifierFilters){
		// if contains expression based filter
		if(entry.getKeyType() == TokenType.EXP
				|| entry.getValueType() == TokenType.EXP
				|| entry.getKeyType() != TokenType.ID){
			if(!EagleConfigFactory.load().isCoprocessorEnabled()) {
				LOG.warn("Expression in filter may not support, because custom filter and coprocessor is disabled: " + entry.toString());
			}
			list.addFilter(buildExpressionBasedFilter(entry));
			continue;
		}

		// else using SingleColumnValueFilter
		String qualifierName = entry.getKey();
		if(!isTag(entry.getKey())){
			Qualifier qualifier = _ed.getDisplayNameMap().get(entry.getKey());
			qualifierName = qualifier.getQualifierName();
		}

		// Comparator to be used for building HBase Filter
		// WritableByteArrayComparable comparator;
           ByteArrayComparable comparable;
		if(ComparisonOperator.IN.equals(entry.getOp())
			|| ComparisonOperator.NOT_IN.equals(entry.getOp())){
			Filter setFilter = buildListQualifierFilter(entry);
			if(setFilter!=null){
				list.addFilter(setFilter);
			}
		}else{
			// If [=,!=,is,is not] NULL, use NullComparator else throw exception
			if(TokenType.NULL.equals(entry.getValueType())){
				if(ComparisonOperator.EQUAL.equals(entry.getOp())
					||ComparisonOperator.NOT_EQUAL.equals(entry.getOp())
					||ComparisonOperator.IS.equals(entry.getOp())
					||ComparisonOperator.IS_NOT.equals(entry.getOp()))
                       comparable = new NullComparator();
				else
					throw new IllegalArgumentException("Operation: "+entry.getOp()+" with NULL is not supported yet: "+entry.toString()+", avaliable options: [=, !=, is, is not] null|NULL");
			}
			// If [contains, not contains],use SubstringComparator
			else if (ComparisonOperator.CONTAINS.equals(entry.getOp())
				|| ComparisonOperator.NOT_CONTAINS.equals(entry.getOp())) {
                   comparable = new SubstringComparator(entry.getValue());
			}
			// If [like, not like], use RegexStringComparator
			else if (ComparisonOperator.LIKE.equals(entry.getOp())
					|| ComparisonOperator.NOT_LIKE.equals(entry.getOp())){
				// Use RegexStringComparator for LIKE / NOT_LIKE
				RegexStringComparator _comparator = new RegexStringComparator(buildQualifierRegex(entry.getValue()));
				_comparator.setCharset(_charset);
                   comparable = _comparator;
			} else{
				Class type = EntityQualifierUtils.getType(_ed, entry.getKey());
				// if type is null (is Tag or not found) or not defined for TypedByteArrayComparator
				if(!EagleConfigFactory.load().isCoprocessorEnabled() || type == null || TypedByteArrayComparator.get(type) == null){
                       comparable = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()));
				}else {
                       comparable = new TypedByteArrayComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()),type);
				}
			}

			SingleColumnValueFilter filter =
					new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparable);
			filter.setFilterIfMissing(_filterIfMissing);
			list.addFilter(filter);
		}
	}

	return list;
}
 
Example 29
Source Project: Eagle   Source File: HBaseFilterBuilder.java    License: Apache License 2.0 4 votes vote down vote up
/**
	 * Currently use BinaryComparator only
	 * <h2>TODO: </h2>
	 * Possibility to tune performance by using: OR[BinaryComparator,...] instead of RegexStringComparator?
	 *
	 *<br/> <br/>
	 *
	 * ! Check op must be IN or NOTIN in caller
	 *
	 * @param entry
	 * @return
	 */
	private Filter buildListQualifierFilter(QualifierFilterEntity entry){
		List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
		Iterator<String> it = valueSet.iterator();
		String fieldName = entry.getKey();
		String qualifierName = fieldName;
		if(!_ed.isTag(entry.getKey())){
			qualifierName = _ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
		}

// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR list.
//		Class qualifierType = EntityQualifierHelper.getType(_ed,fieldName);
//		if( qualifierType == null || qualifierType == String.class){
//			boolean first = true;
//			StringBuilder filterRegex = new StringBuilder();
//			filterRegex.append("^(");
//			while(it.hasNext()) {
//				String value = it.next();
//				if(value == null) {
//					logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
//					continue;
//				}
//				if(!first) filterRegex.append("|");
//				filterRegex.append(value);
//				first = false;
//			}
//			filterRegex.append(")$");
//			RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
//			return new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
//					convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
//		}else{
		FilterList setFilterList;
		if(ComparisonOperator.IN.equals(entry.getOp())){
			setFilterList = new FilterList(Operator.MUST_PASS_ONE);
		}else if(ComparisonOperator.NOT_IN.equals(entry.getOp())) {
			setFilterList = new FilterList(Operator.MUST_PASS_ALL);
		}else{
			throw new IllegalArgumentException(String.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",entry.getOp(),entry.toString()));
		}

		while(it.hasNext()) {
			String value = it.next();
			BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, fieldName, value));
			SingleColumnValueFilter filter =
					new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
			filter.setFilterIfMissing(_filterIfMissing);
			setFilterList.addFilter(filter);
		}

		return setFilterList;
//		}
	}
 
Example 30
Source Project: eagle   Source File: HBaseFilterBuilder.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * @see org.apache.eagle.query.parser.TokenType
 * @return
 */
public FilterList buildFilters() {
    // TODO: Optimize to select between row filter or column filter for better performance
    // Use row key filter priority by default
    boolean rowFilterPriority = true;

    FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
    for (ANDExpression andExpr : orExpr.getANDExprList()) {

        FilterList list = new FilterList(Operator.MUST_PASS_ALL);
        Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
        List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
        // List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();

        // TODO refactor not to use too much if/else
        for (AtomicExpression ae : andExpr.getAtomicExprList()) {
            // TODO temporarily ignore those fields which are not for attributes

            String fieldName = ae.getKey();
            if (ae.getKeyType() == TokenType.ID) {
                fieldName = parseEntityAttribute(fieldName);
                if (fieldName == null) {
                    LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
                    continue;
                }
            }

            String value = ae.getValue();
            ComparisonOperator op = ae.getOp();
            TokenType keyType = ae.getKeyType();
            TokenType valueType = ae.getValueType();
            QualifierFilterEntity entry = new QualifierFilterEntity(fieldName, value, op, keyType,
                                                                    valueType);

            // TODO Exact match, need to add escape for those special characters here, including:
            // "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"

            if (keyType == TokenType.ID && isTag(fieldName)) {
                if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
                    && !TokenType.NULL.equals(valueType)) {
                    // Use RowFilter for equal TAG
                    if (tagFilters.get(fieldName) == null) {
                        tagFilters.put(fieldName, new ArrayList<String>());
                    }
                    tagFilters.get(fieldName).add(value);
                } else if (rowFilterPriority && ComparisonOperator.IN.equals(op)) {
                    // Use RowFilter here by default
                    if (tagFilters.get(fieldName) == null) {
                        tagFilters.put(fieldName, new ArrayList<String>());
                    }
                    tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
                } else if (ComparisonOperator.LIKE.equals(op) || ComparisonOperator.NOT_LIKE.equals(op)
                           || ComparisonOperator.CONTAINS.equals(op)
                           || ComparisonOperator.NOT_CONTAINS.equals(op)
                           || ComparisonOperator.IN.equals(op) || ComparisonOperator.IS.equals(op)
                           || ComparisonOperator.IS_NOT.equals(op)
                           || ComparisonOperator.NOT_EQUAL.equals(op)
                           || ComparisonOperator.EQUAL.equals(op)
                           || ComparisonOperator.NOT_IN.equals(op)) {
                    qualifierFilters.add(entry);
                } else {
                    LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName
                             + " yet, going to ignore");
                    throw new IllegalArgumentException("Don't support operation: " + op
                                                       + " on tag field: " + fieldName
                                                       + ", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
                }
            } else {
                qualifierFilters.add(entry);
            }
        }

        // Build RowFilter for equal tags
        list.addFilter(buildTagFilter(tagFilters));

        // Build SingleColumnValueFilter
        FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
        if (qualifierFilterList != null && qualifierFilterList.getFilters().size() > 0) {
            list.addFilter(qualifierFilterList);
        } else {
            if (LOG.isDebugEnabled()) {
                LOG.debug("Ignore empty qualifier filter from " + qualifierFilters.toString());
            }
        }
        fltList.addFilter(list);
    }
    LOG.info("Query: " + orExpr.toString() + " => Filter: " + fltList.toString());
    return fltList;
}