Java Code Examples for org.apache.hadoop.hbase.filter.FilterList.Operator#MUST_PASS_ONE
The following examples show how to use
org.apache.hadoop.hbase.filter.FilterList.Operator#MUST_PASS_ONE .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 6 votes |
|
@Test
public void testWithMultiVersionsInSameRow() throws Exception {
FilterList filterList01 =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 0));
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
1, Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
2, Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual"),
3, Bytes.toBytes("value"));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList01.filterCell(kv1));
assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv2));
assertEquals(ReturnCode.NEXT_COL, filterList01.filterCell(kv3));
FilterList filterList11 =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, 1));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv1));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv2));
assertEquals(ReturnCode.NEXT_COL, filterList11.filterCell(kv3));
}
Example 2
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 6 votes |
|
@Test
public void testMPONEWithSeekNextUsingHint() throws Exception {
byte[] col = Bytes.toBytes("c");
FilterList filterList =
new FilterList(Operator.MUST_PASS_ONE, new ColumnPaginationFilter(1, col));
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 2,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 3,
Bytes.toBytes("value"));
KeyValue kv4 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
Bytes.toBytes("value"));
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv2));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3));
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv4));
}
Example 3
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testTransformCell() throws IOException {
KeyValue kv =
new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), 1,
Bytes.toBytes("value"));
// case MUST_PASS_ONE
TransformFilter filter1 = new TransformFilter(ReturnCode.INCLUDE);
TransformFilter filter2 = new TransformFilter(ReturnCode.NEXT_ROW);
TransformFilter filter3 = new TransformFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(false, filter2.getTransformed());
Assert.assertEquals(false, filter3.getTransformed());
// case MUST_PASS_ALL
filter1 = new TransformFilter(ReturnCode.INCLUDE);
filter2 = new TransformFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
filter3 = new TransformFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(true, filter2.getTransformed());
Assert.assertEquals(true, filter3.getTransformed());
}
Example 4
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testRowCountFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam1"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam2"), Bytes.toBytes("a"), 2,
Bytes.toBytes("value"));
MockNextRowFilter mockNextRowFilter = new MockNextRowFilter();
FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockNextRowFilter);
filter.filterCell(kv1);
filter.filterCell(kv2);
assertEquals(2, mockNextRowFilter.getHitCount());
}
Example 5
| Source File: Filters.java From java-docs-samples with Apache License 2.0 | 5 votes |
|
public static void filterComposingInterleave(
String projectId, String instanceId, String tableId) {
// A filter that matches cells with the value true OR with the column qualifier os_build
Filter qualifierFilter =
new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("os_build")));
Filter valueFilter =
new ValueFilter(CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("true")));
FilterList filter = new FilterList(Operator.MUST_PASS_ONE);
filter.addFilter(qualifierFilter);
filter.addFilter(valueFilter);
Scan scan = new Scan().setFilter(filter).setMaxVersions();
readWithFilter(projectId, instanceId, tableId, scan);
}
Example 6
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testEmptyFilterListTransformCell() throws IOException {
KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value"));
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
filterList = new FilterList(Operator.MUST_PASS_ONE);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
}
Example 7
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testKeyOnlyFilterTransformCell() throws IOException {
Cell c;
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value1"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
2, Bytes.toBytes("value2"));
Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value1"));
Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value2"));
FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
FilterList keyOnlyFilterFirst =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv1));
c = keyOnlyFilterFirst.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv2));
c = keyOnlyFilterFirst.transformCell(kv2);
assertEquals(0, c.getValueLength());
internalFilterList.reset();
FilterList keyOnlyFilterLast =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv1));
c = keyOnlyFilterLast.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv2));
c = keyOnlyFilterLast.transformCell(kv2);
assertEquals(0, c.getValueLength());
}
Example 8
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
/**
* Tests the behavior of transform() in a hierarchical filter.
*
* transform() only applies after a filterCell() whose return-code includes the KeyValue.
* Lazy evaluation of AND
*/
@Test
public void testTransformMPO() throws Exception {
// Apply the following filter:
// (family=fam AND qualifier=qual1 AND KeyOnlyFilter)
// OR (family=fam AND qualifier=qual2)
final FilterList flist = new FilterList(Operator.MUST_PASS_ONE, Lists.<Filter>newArrayList(
new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual1"))),
new KeyOnlyFilter())),
new FilterList(Operator.MUST_PASS_ALL, Lists.<Filter>newArrayList(
new FamilyFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("fam"))),
new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(Bytes.toBytes("qual2")))))));
final KeyValue kvQual1 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual1"), Bytes.toBytes("value"));
final KeyValue kvQual2 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual2"), Bytes.toBytes("value"));
final KeyValue kvQual3 = new KeyValue(
Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("qual3"), Bytes.toBytes("value"));
// Value for fam:qual1 should be stripped:
assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual1));
final KeyValue transformedQual1 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual1));
assertEquals(0, transformedQual1.getValueLength());
// Value for fam:qual2 should not be stripped:
assertEquals(Filter.ReturnCode.INCLUDE, flist.filterCell(kvQual2));
final KeyValue transformedQual2 = KeyValueUtil.ensureKeyValue(flist.transformCell(kvQual2));
assertEquals("value", Bytes.toString(transformedQual2.getValueArray(),
transformedQual2.getValueOffset(), transformedQual2.getValueLength()));
// Other keys should be skipped:
assertEquals(Filter.ReturnCode.SKIP, flist.filterCell(kvQual3));
}
Example 9
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
Filter filter1 = new MockSeekHintFilter(kv1);
filter1.setReversed(true);
Filter filter2 = new MockSeekHintFilter(kv2);
filter2.setReversed(true);
Filter filter3 = new MockSeekHintFilter(kv3);
filter3.setReversed(true);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
}
Example 10
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testTheMinimalRule() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2, filter3);
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter5, filter6);
assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter4, filter6);
assertEquals(ReturnCode.SKIP, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter1);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter3, filter2, filter1, filter5);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter2,
new FilterList(Operator.MUST_PASS_ONE, filter3, filter4));
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ONE, filter6, filter6);
assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
}
Example 11
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 4 votes |
|
@Test
public void testFilterListWithORWhenPassingCellMismatchPreviousRC() throws IOException {
// Mainly test FilterListWithOR#calculateReturnCodeByPrevCellAndRC method with two sub-filters.
KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"),
100, Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 99,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 1,
Bytes.toBytes("value"));
KeyValue kv4 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fan"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
Filter subFilter1 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL);
Mockito.when(subFilter1.filterCell(kv2)).thenReturn(ReturnCode.NEXT_COL);
Mockito.when(subFilter1.filterCell(kv3)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL);
Mockito.when(subFilter1.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_NEXT_COL);
Filter subFilter2 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SKIP);
Mockito.when(subFilter2.filterCell(kv2)).thenReturn(ReturnCode.NEXT_ROW);
Mockito.when(subFilter2.filterCell(kv3)).thenReturn(ReturnCode.NEXT_ROW);
Mockito.when(subFilter2.filterCell(kv4)).thenReturn(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
Filter filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2);
Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv1));
Assert.assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv2));
Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv3));
Assert.assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv4));
// One sub-filter will filterAllRemaining but other sub-filter will return SEEK_HINT
subFilter1 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter1.filterAllRemaining()).thenReturn(true);
Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.NEXT_ROW);
subFilter2 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT);
filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
// Two sub-filter returns SEEK_NEXT_USING_HINT, then we should return SEEK_NEXT_USING_HINT.
subFilter1 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter1.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT);
subFilter2 = Mockito.mock(FilterBase.class);
Mockito.when(subFilter2.filterCell(kv1)).thenReturn(ReturnCode.SEEK_NEXT_USING_HINT);
filterList = new FilterList(Operator.MUST_PASS_ONE, subFilter1, subFilter2);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
}
Example 12
| Source File: HBaseFilterBuilder.java From Eagle with Apache License 2.0 | 4 votes |
|
/**
* @see org.apache.eagle.query.parser.TokenType
*
* @return
*/
public FilterList buildFilters(){
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for(ANDExpression andExpr : _orExpr.getANDExprList()){
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for(AtomicExpression ae : andExpr.getAtomicExprList()){
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if(ae.getKeyType() == TokenType.ID){
fieldName = parseEntityAttribute(fieldName);
if(fieldName == null){
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName,value,op,keyType,valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if(keyType == TokenType.ID && isTag(fieldName)){
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType))
{
// Use RowFilter for equal TAG
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op))
{
// Use RowFilter here by default
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op)
|| ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op)
|| ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op))
{
qualifierFilters.add(entry);
} else
{
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName + " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: "+op+" on tag field: "+fieldName+", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
}else{
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if(qualifierFilterList != null && qualifierFilterList.getFilters().size()>0){
list.addFilter(qualifierFilterList);
}else {
if(LOG.isDebugEnabled()) LOG.debug("Ignore empty qualifier filter from "+qualifierFilters.toString());
}
fltList.addFilter(list);
}
LOG.info("Query: " + _orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}
Example 13
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 4 votes |
|
@Test
public void testShouldPassCurrentCellToFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 2,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("b"), 3,
Bytes.toBytes("value"));
KeyValue kv4 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("c"), 4,
Bytes.toBytes("value"));
MockFilter mockFilter = new MockFilter(ReturnCode.NEXT_COL);
FilterList filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
filter.filterCell(kv1);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv2);
assertFalse(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv3);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
filter.filterCell(kv1);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv2);
assertFalse(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv3);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter = new MockFilter(ReturnCode.NEXT_ROW);
filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
filter.filterCell(kv1);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv2);
assertFalse(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv3);
assertFalse(mockFilter.didCellPassToTheFilter);
filter.reset();
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv4);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
filter = new FilterList(Operator.MUST_PASS_ONE, mockFilter);
filter.filterCell(kv1);
assertTrue(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv2);
assertFalse(mockFilter.didCellPassToTheFilter);
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv3);
assertFalse(mockFilter.didCellPassToTheFilter);
filter.reset();
mockFilter.didCellPassToTheFilter = false;
filter.filterCell(kv4);
assertTrue(mockFilter.didCellPassToTheFilter);
}
Example 14
| Source File: TestFilter.java From hbase with Apache License 2.0 | 4 votes |
|
@Test
public void testFilterListWithPrefixFilter() throws IOException {
byte[] family = Bytes.toBytes("f1");
byte[] qualifier = Bytes.toBytes("q1");
TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
new TableDescriptorBuilder.ModifyableTableDescriptor(
TableName.valueOf(name.getMethodName()));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
HRegion testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
TEST_UTIL.getConfiguration(), tableDescriptor);
for(int i=0; i<5; i++) {
Put p = new Put(Bytes.toBytes((char)('a'+i) + "row"));
p.setDurability(Durability.SKIP_WAL);
p.addColumn(family, qualifier, Bytes.toBytes(String.valueOf(111 + i)));
testRegion.put(p);
}
testRegion.flush(true);
// rows starting with "b"
PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ;
// rows with value of column 'q1' set to '113'
SingleColumnValueFilter scvf = new SingleColumnValueFilter(
family, qualifier, CompareOperator.EQUAL, Bytes.toBytes("113"));
// combine these two with OR in a FilterList
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf);
Scan s1 = new Scan();
s1.setFilter(filterList);
InternalScanner scanner = testRegion.getScanner(s1);
List<Cell> results = new ArrayList<>();
int resultCount = 0;
while (scanner.next(results)) {
resultCount++;
byte[] row = CellUtil.cloneRow(results.get(0));
LOG.debug("Found row: " + Bytes.toStringBinary(row));
assertTrue(Bytes.equals(row, Bytes.toBytes("brow"))
|| Bytes.equals(row, Bytes.toBytes("crow")));
results.clear();
}
assertEquals(2, resultCount);
scanner.close();
WAL wal = ((HRegion)testRegion).getWAL();
((HRegion)testRegion).close();
wal.close();
}
Example 15
| Source File: TestFilter.java From hbase with Apache License 2.0 | 4 votes |
|
@Test
public void testFilterListWithSingleColumnValueFilter() throws IOException {
// Test for HBASE-3191
// Scan using SingleColumnValueFilter
SingleColumnValueFilter f1 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0],
CompareOperator.EQUAL, VALUES[0]);
f1.setFilterIfMissing( true );
Scan s1 = new Scan();
s1.addFamily(FAMILIES[0]);
s1.setFilter(f1);
KeyValue [] kvs1 = {
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
};
verifyScanNoEarlyOut(s1, 3, 3);
verifyScanFull(s1, kvs1);
// Scan using another SingleColumnValueFilter, expect disjoint result
SingleColumnValueFilter f2 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_TWO[0],
CompareOperator.EQUAL, VALUES[1]);
f2.setFilterIfMissing( true );
Scan s2 = new Scan();
s2.addFamily(FAMILIES[0]);
s2.setFilter(f2);
KeyValue [] kvs2 = {
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanNoEarlyOut(s2, 3, 3);
verifyScanFull(s2, kvs2);
// Scan, ORing the two previous filters, expect unified result
FilterList f = new FilterList(Operator.MUST_PASS_ONE);
f.addFilter(f1);
f.addFilter(f2);
Scan s = new Scan();
s.addFamily(FAMILIES[0]);
s.setFilter(f);
KeyValue [] kvs = {
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanNoEarlyOut(s, 6, 3);
verifyScanFull(s, kvs);
}
Example 16
| Source File: HBaseFilterBuilder.java From eagle with Apache License 2.0 | 4 votes |
|
/**
* Currently use BinaryComparator only
* <h2>TODO:</h2> Possibility to tune performance by using: OR[BinaryComparator,...] instead of
* RegexStringComparator? <br/>
* <br/>
* ! Check op must be IN or NOTIN in caller
*
* @param entry
* @return
*/
private Filter buildListQualifierFilter(QualifierFilterEntity entry) {
List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
Iterator<String> it = valueSet.iterator();
String fieldName = entry.getKey();
String qualifierName = fieldName;
if (!ed.isTag(entry.getKey())) {
qualifierName = ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
}
// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR
// list.
// Class qualifierType = EntityQualifierHelper.getType(ed,fieldName);
// if( qualifierType == null || qualifierType == String.class){
// boolean first = true;
// StringBuilder filterRegex = new StringBuilder();
// filterRegex.append("^(");
// while(it.hasNext()) {
// String value = it.next();
// if(value == null) {
// logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
// continue;
// }
// if(!first) filterRegex.append("|");
// filterRegex.append(value);
// first = false;
// }
// filterRegex.append(")$");
// RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
// return new SingleColumnValueFilter(ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
// convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
// }else{
FilterList setFilterList;
if (ComparisonOperator.IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ONE);
} else if (ComparisonOperator.NOT_IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ALL);
} else {
throw new IllegalArgumentException(String
.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",
entry.getOp(), entry.toString()));
}
while (it.hasNext()) {
String value = it.next();
BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(ed, fieldName,
value));
SingleColumnValueFilter filter = new SingleColumnValueFilter(ed.getColumnFamily()
.getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
filter.setFilterIfMissing(filterIfMissing);
setFilterList.addFilter(filter);
}
return setFilterList;
// }
}
Example 17
| Source File: HBaseFilterBuilder.java From eagle with Apache License 2.0 | 4 votes |
|
/**
* @see org.apache.eagle.query.parser.TokenType
* @return
*/
public FilterList buildFilters() {
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for (ANDExpression andExpr : orExpr.getANDExprList()) {
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for (AtomicExpression ae : andExpr.getAtomicExprList()) {
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if (ae.getKeyType() == TokenType.ID) {
fieldName = parseEntityAttribute(fieldName);
if (fieldName == null) {
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName, value, op, keyType,
valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if (keyType == TokenType.ID && isTag(fieldName)) {
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType)) {
// Use RowFilter for equal TAG
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op)) {
// Use RowFilter here by default
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op) || ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op) || ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op)) {
qualifierFilters.add(entry);
} else {
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName
+ " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: " + op
+ " on tag field: " + fieldName
+ ", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
} else {
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if (qualifierFilterList != null && qualifierFilterList.getFilters().size() > 0) {
list.addFilter(qualifierFilterList);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignore empty qualifier filter from " + qualifierFilters.toString());
}
}
fltList.addFilter(list);
}
LOG.info("Query: " + orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}
Example 18
| Source File: HBaseFilterBuilder.java From Eagle with Apache License 2.0 | 4 votes |
|
/**
* Currently use BinaryComparator only
* <h2>TODO: </h2>
* Possibility to tune performance by using: OR[BinaryComparator,...] instead of RegexStringComparator?
*
*<br/> <br/>
*
* ! Check op must be IN or NOTIN in caller
*
* @param entry
* @return
*/
private Filter buildListQualifierFilter(QualifierFilterEntity entry){
List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
Iterator<String> it = valueSet.iterator();
String fieldName = entry.getKey();
String qualifierName = fieldName;
if(!_ed.isTag(entry.getKey())){
qualifierName = _ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
}
// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR list.
// Class qualifierType = EntityQualifierHelper.getType(_ed,fieldName);
// if( qualifierType == null || qualifierType == String.class){
// boolean first = true;
// StringBuilder filterRegex = new StringBuilder();
// filterRegex.append("^(");
// while(it.hasNext()) {
// String value = it.next();
// if(value == null) {
// logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
// continue;
// }
// if(!first) filterRegex.append("|");
// filterRegex.append(value);
// first = false;
// }
// filterRegex.append(")$");
// RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
// return new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
// convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
// }else{
FilterList setFilterList;
if(ComparisonOperator.IN.equals(entry.getOp())){
setFilterList = new FilterList(Operator.MUST_PASS_ONE);
}else if(ComparisonOperator.NOT_IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ALL);
}else{
throw new IllegalArgumentException(String.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",entry.getOp(),entry.toString()));
}
while(it.hasNext()) {
String value = it.next();
BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, fieldName, value));
SingleColumnValueFilter filter =
new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
filter.setFilterIfMissing(_filterIfMissing);
setFilterList.addFilter(filter);
}
return setFilterList;
// }
}
