Java Code Examples for org.apache.hadoop.hbase.filter.FilterList.Operator#MUST_PASS_ALL
The following examples show how to use
org.apache.hadoop.hbase.filter.FilterList.Operator#MUST_PASS_ALL .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: DefaultHBaseSerde.java From envelope with Apache License 2.0 | 6 votes |
|
private FilterList getColumnValueFilters(Row row) {
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
Set<String> filterColumnNames = Sets.newHashSet(row.schema().fieldNames());
for (Map.Entry<String, ColumnDef> column : columns.entrySet()) {
if (!column.getValue().cf.equals("rowkey")) {
if (filterColumnNames.contains(column.getKey())) {
byte[] value = getColumnValueAsBytes(column.getValue().name, column.getValue().type, row);
if (value != null) {
SingleColumnValueFilter columnValueFilter = new SingleColumnValueFilter(
Bytes.toBytes(column.getValue().cf),
Bytes.toBytes(column.getValue().name),
CompareFilter.CompareOp.EQUAL,
value
);
filterList.addFilter(columnValueFilter);
}
}
}
}
return filterList;
}
Example 2
| Source File: TestFuzzyRowFilterEndToEnd.java From hbase with Apache License 2.0 | 5 votes |
|
private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
throws IOException {
String cf = "f";
Scan scan = new Scan();
scan.addFamily(Bytes.toBytes(cf));
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
scan.setFilter(filterList);
ResultScanner scanner = hTable.getScanner(scan);
List<Cell> results = new ArrayList<>();
Result result;
long timeBeforeScan = System.currentTimeMillis();
while ((result = scanner.next()) != null) {
for (Cell kv : result.listCells()) {
LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
+ Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
results.add(kv);
}
}
long scanTime = System.currentTimeMillis() - timeBeforeScan;
scanner.close();
LOG.info("scan time = " + scanTime + "ms");
LOG.info("found " + results.size() + " results");
assertEquals(expectedSize, results.size());
}
Example 3
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testAddFilter() throws Exception {
Filter filter1 = new FirstKeyOnlyFilter();
Filter filter2 = new FirstKeyOnlyFilter();
FilterList filterList = new FilterList(filter1, filter2);
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Arrays.asList(filter1, filter2));
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
filterList.addFilter(new FirstKeyOnlyFilter());
filterList = new FilterList(Operator.MUST_PASS_ALL, Arrays.asList(filter1, filter2));
filterList.addFilter(new FirstKeyOnlyFilter());
filterList.setReversed(false);
FirstKeyOnlyFilter f = new FirstKeyOnlyFilter();
f.setReversed(true);
try {
filterList.addFilter(f);
fail("The IllegalArgumentException should be thrown because the added filter is reversed");
} catch (IllegalArgumentException e) {
}
}
Example 4
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testTheMaximalRule() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
MockFilter filter1 = new MockFilter(ReturnCode.INCLUDE);
MockFilter filter2 = new MockFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
MockFilter filter3 = new MockFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
MockFilter filter4 = new MockFilter(ReturnCode.NEXT_COL);
MockFilter filter5 = new MockFilter(ReturnCode.SKIP);
MockFilter filter6 = new MockFilter(ReturnCode.SEEK_NEXT_USING_HINT);
MockFilter filter7 = new MockFilter(ReturnCode.NEXT_ROW);
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
assertEquals(ReturnCode.INCLUDE_AND_NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter2, filter3);
assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter5, filter6);
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter4, filter6);
assertEquals(ReturnCode.NEXT_COL, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter1);
assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter2, filter1, filter5);
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter2,
new FilterList(Operator.MUST_PASS_ALL, filter3, filter4));
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL, filter3, filter7);
assertEquals(ReturnCode.NEXT_ROW, filterList.filterCell(kv1));
}
Example 5
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testReversedFilterListWithMockSeekHintFilter() throws IOException {
KeyValue kv1 = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row2"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
KeyValue kv3 = new KeyValue(Bytes.toBytes("row3"), Bytes.toBytes("fam"), Bytes.toBytes("a"), 1,
Bytes.toBytes("value"));
Filter filter1 = new MockSeekHintFilter(kv1);
filter1.setReversed(true);
Filter filter2 = new MockSeekHintFilter(kv2);
filter2.setReversed(true);
Filter filter3 = new MockSeekHintFilter(kv3);
filter3.setReversed(true);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv3, filterList.getNextCellHint(kv1));
filterList = new FilterList(Operator.MUST_PASS_ALL);
filterList.setReversed(true);
filterList.addFilter(filter1);
filterList.addFilter(filter2);
filterList.addFilter(filter3);
Assert.assertEquals(ReturnCode.SEEK_NEXT_USING_HINT, filterList.filterCell(kv1));
Assert.assertEquals(kv1, filterList.getNextCellHint(kv1));
}
Example 6
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testKeyOnlyFilterTransformCell() throws IOException {
Cell c;
KeyValue kv1 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value1"));
KeyValue kv2 = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
2, Bytes.toBytes("value2"));
Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value1"));
Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes("cf"), Bytes.toBytes("column1"),
CompareOperator.EQUAL, Bytes.toBytes("value2"));
FilterList internalFilterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2);
FilterList keyOnlyFilterFirst =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv1));
c = keyOnlyFilterFirst.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterFirst.filterCell(kv2));
c = keyOnlyFilterFirst.transformCell(kv2);
assertEquals(0, c.getValueLength());
internalFilterList.reset();
FilterList keyOnlyFilterLast =
new FilterList(Operator.MUST_PASS_ALL, new KeyOnlyFilter(), internalFilterList);
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv1));
c = keyOnlyFilterLast.transformCell(kv1);
assertEquals(0, c.getValueLength());
assertEquals(ReturnCode.INCLUDE, keyOnlyFilterLast.filterCell(kv2));
c = keyOnlyFilterLast.transformCell(kv2);
assertEquals(0, c.getValueLength());
}
Example 7
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testEmptyFilterListTransformCell() throws IOException {
KeyValue kv = new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"),
1, Bytes.toBytes("value"));
FilterList filterList = new FilterList(Operator.MUST_PASS_ALL);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
filterList = new FilterList(Operator.MUST_PASS_ONE);
assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
assertEquals(kv, filterList.transformCell(kv));
}
Example 8
| Source File: TestFilterList.java From hbase with Apache License 2.0 | 5 votes |
|
@Test
public void testTransformCell() throws IOException {
KeyValue kv =
new KeyValue(Bytes.toBytes("row"), Bytes.toBytes("cf"), Bytes.toBytes("column1"), 1,
Bytes.toBytes("value"));
// case MUST_PASS_ONE
TransformFilter filter1 = new TransformFilter(ReturnCode.INCLUDE);
TransformFilter filter2 = new TransformFilter(ReturnCode.NEXT_ROW);
TransformFilter filter3 = new TransformFilter(ReturnCode.SEEK_NEXT_USING_HINT);
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(false, filter2.getTransformed());
Assert.assertEquals(false, filter3.getTransformed());
// case MUST_PASS_ALL
filter1 = new TransformFilter(ReturnCode.INCLUDE);
filter2 = new TransformFilter(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW);
filter3 = new TransformFilter(ReturnCode.INCLUDE_AND_NEXT_COL);
filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2, filter3);
Assert.assertEquals(ReturnCode.INCLUDE_AND_SEEK_NEXT_ROW, filterList.filterCell(kv));
Assert.assertEquals(kv, filterList.transformCell(kv));
Assert.assertEquals(true, filter1.getTransformed());
Assert.assertEquals(true, filter2.getTransformed());
Assert.assertEquals(true, filter3.getTransformed());
}
Example 9
| Source File: HBaseFilterBuilder.java From Eagle with Apache License 2.0 | 4 votes |
|
/**
* @see org.apache.eagle.query.parser.TokenType
*
* @return
*/
public FilterList buildFilters(){
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for(ANDExpression andExpr : _orExpr.getANDExprList()){
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for(AtomicExpression ae : andExpr.getAtomicExprList()){
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if(ae.getKeyType() == TokenType.ID){
fieldName = parseEntityAttribute(fieldName);
if(fieldName == null){
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName,value,op,keyType,valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if(keyType == TokenType.ID && isTag(fieldName)){
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType))
{
// Use RowFilter for equal TAG
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op))
{
// Use RowFilter here by default
if(tagFilters.get(fieldName) == null) tagFilters.put(fieldName, new ArrayList<String>());
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op)
|| ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op)
|| ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op))
{
qualifierFilters.add(entry);
} else
{
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName + " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: "+op+" on tag field: "+fieldName+", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
}else{
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if(qualifierFilterList != null && qualifierFilterList.getFilters().size()>0){
list.addFilter(qualifierFilterList);
}else {
if(LOG.isDebugEnabled()) LOG.debug("Ignore empty qualifier filter from "+qualifierFilters.toString());
}
fltList.addFilter(list);
}
LOG.info("Query: " + _orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}
Example 10
| Source File: HBaseFilterBuilder.java From Eagle with Apache License 2.0 | 4 votes |
|
/**
* all qualifiers' condition must be satisfied.
*
* <H1>Use RegexStringComparator for:</H1>
* IN
* LIKE
* NOT_LIKE
*
* <H1>Use SubstringComparator for:</H1>
* CONTAINS
*
* <H1>Use EntityQualifierHelper for:</H1>
* EQUALS
* NOT_EUQALS
* LESS
* LESS_OR_EQUAL
* GREATER
* GREATER_OR_EQUAL
*
* <H2>
* TODO: Compare performance of RegexStringComparator ,SubstringComparator ,EntityQualifierHelper
* </H2>
*
* @param qualifierFilters
* @return
*/
protected FilterList buildQualifierFilter(List<QualifierFilterEntity> qualifierFilters){
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
// iterate all the qualifiers
for(QualifierFilterEntity entry : qualifierFilters){
// if contains expression based filter
if(entry.getKeyType() == TokenType.EXP
|| entry.getValueType() == TokenType.EXP
|| entry.getKeyType() != TokenType.ID){
if(!EagleConfigFactory.load().isCoprocessorEnabled()) {
LOG.warn("Expression in filter may not support, because custom filter and coprocessor is disabled: " + entry.toString());
}
list.addFilter(buildExpressionBasedFilter(entry));
continue;
}
// else using SingleColumnValueFilter
String qualifierName = entry.getKey();
if(!isTag(entry.getKey())){
Qualifier qualifier = _ed.getDisplayNameMap().get(entry.getKey());
qualifierName = qualifier.getQualifierName();
}
// Comparator to be used for building HBase Filter
// WritableByteArrayComparable comparator;
ByteArrayComparable comparable;
if(ComparisonOperator.IN.equals(entry.getOp())
|| ComparisonOperator.NOT_IN.equals(entry.getOp())){
Filter setFilter = buildListQualifierFilter(entry);
if(setFilter!=null){
list.addFilter(setFilter);
}
}else{
// If [=,!=,is,is not] NULL, use NullComparator else throw exception
if(TokenType.NULL.equals(entry.getValueType())){
if(ComparisonOperator.EQUAL.equals(entry.getOp())
||ComparisonOperator.NOT_EQUAL.equals(entry.getOp())
||ComparisonOperator.IS.equals(entry.getOp())
||ComparisonOperator.IS_NOT.equals(entry.getOp()))
comparable = new NullComparator();
else
throw new IllegalArgumentException("Operation: "+entry.getOp()+" with NULL is not supported yet: "+entry.toString()+", avaliable options: [=, !=, is, is not] null|NULL");
}
// If [contains, not contains],use SubstringComparator
else if (ComparisonOperator.CONTAINS.equals(entry.getOp())
|| ComparisonOperator.NOT_CONTAINS.equals(entry.getOp())) {
comparable = new SubstringComparator(entry.getValue());
}
// If [like, not like], use RegexStringComparator
else if (ComparisonOperator.LIKE.equals(entry.getOp())
|| ComparisonOperator.NOT_LIKE.equals(entry.getOp())){
// Use RegexStringComparator for LIKE / NOT_LIKE
RegexStringComparator _comparator = new RegexStringComparator(buildQualifierRegex(entry.getValue()));
_comparator.setCharset(_charset);
comparable = _comparator;
} else{
Class type = EntityQualifierUtils.getType(_ed, entry.getKey());
// if type is null (is Tag or not found) or not defined for TypedByteArrayComparator
if(!EagleConfigFactory.load().isCoprocessorEnabled() || type == null || TypedByteArrayComparator.get(type) == null){
comparable = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()));
}else {
comparable = new TypedByteArrayComparator(EntityQualifierUtils.toBytes(_ed, entry.getKey(), entry.getValue()),type);
}
}
SingleColumnValueFilter filter =
new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparable);
filter.setFilterIfMissing(_filterIfMissing);
list.addFilter(filter);
}
}
return list;
}
Example 11
| Source File: HBaseFilterBuilder.java From Eagle with Apache License 2.0 | 4 votes |
|
/**
* Currently use BinaryComparator only
* <h2>TODO: </h2>
* Possibility to tune performance by using: OR[BinaryComparator,...] instead of RegexStringComparator?
*
*<br/> <br/>
*
* ! Check op must be IN or NOTIN in caller
*
* @param entry
* @return
*/
private Filter buildListQualifierFilter(QualifierFilterEntity entry){
List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
Iterator<String> it = valueSet.iterator();
String fieldName = entry.getKey();
String qualifierName = fieldName;
if(!_ed.isTag(entry.getKey())){
qualifierName = _ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
}
// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR list.
// Class qualifierType = EntityQualifierHelper.getType(_ed,fieldName);
// if( qualifierType == null || qualifierType == String.class){
// boolean first = true;
// StringBuilder filterRegex = new StringBuilder();
// filterRegex.append("^(");
// while(it.hasNext()) {
// String value = it.next();
// if(value == null) {
// logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
// continue;
// }
// if(!first) filterRegex.append("|");
// filterRegex.append(value);
// first = false;
// }
// filterRegex.append(")$");
// RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
// return new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
// convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
// }else{
FilterList setFilterList;
if(ComparisonOperator.IN.equals(entry.getOp())){
setFilterList = new FilterList(Operator.MUST_PASS_ONE);
}else if(ComparisonOperator.NOT_IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ALL);
}else{
throw new IllegalArgumentException(String.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",entry.getOp(),entry.toString()));
}
while(it.hasNext()) {
String value = it.next();
BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(_ed, fieldName, value));
SingleColumnValueFilter filter =
new SingleColumnValueFilter(_ed.getColumnFamily().getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
filter.setFilterIfMissing(_filterIfMissing);
setFilterList.addFilter(filter);
}
return setFilterList;
// }
}
Example 12
| Source File: HBaseFilterBuilder.java From eagle with Apache License 2.0 | 4 votes |
|
/**
* @see org.apache.eagle.query.parser.TokenType
* @return
*/
public FilterList buildFilters() {
// TODO: Optimize to select between row filter or column filter for better performance
// Use row key filter priority by default
boolean rowFilterPriority = true;
FilterList fltList = new FilterList(Operator.MUST_PASS_ONE);
for (ANDExpression andExpr : orExpr.getANDExprList()) {
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
Map<String, List<String>> tagFilters = new HashMap<String, List<String>>();
List<QualifierFilterEntity> qualifierFilters = new ArrayList<QualifierFilterEntity>();
// List<QualifierFilterEntry> tagLikeQualifierFilters = new ArrayList<QualifierFilterEntry>();
// TODO refactor not to use too much if/else
for (AtomicExpression ae : andExpr.getAtomicExprList()) {
// TODO temporarily ignore those fields which are not for attributes
String fieldName = ae.getKey();
if (ae.getKeyType() == TokenType.ID) {
fieldName = parseEntityAttribute(fieldName);
if (fieldName == null) {
LOG.warn(fieldName + " field does not have format @<FieldName>, ignored");
continue;
}
}
String value = ae.getValue();
ComparisonOperator op = ae.getOp();
TokenType keyType = ae.getKeyType();
TokenType valueType = ae.getValueType();
QualifierFilterEntity entry = new QualifierFilterEntity(fieldName, value, op, keyType,
valueType);
// TODO Exact match, need to add escape for those special characters here, including:
// "-", "[", "]", "/", "{", "}", "(", ")", "*", "+", "?", ".", "\\", "^", "$", "|"
if (keyType == TokenType.ID && isTag(fieldName)) {
if ((ComparisonOperator.EQUAL.equals(op) || ComparisonOperator.IS.equals(op))
&& !TokenType.NULL.equals(valueType)) {
// Use RowFilter for equal TAG
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).add(value);
} else if (rowFilterPriority && ComparisonOperator.IN.equals(op)) {
// Use RowFilter here by default
if (tagFilters.get(fieldName) == null) {
tagFilters.put(fieldName, new ArrayList<String>());
}
tagFilters.get(fieldName).addAll(EntityQualifierUtils.parseList(value));
} else if (ComparisonOperator.LIKE.equals(op) || ComparisonOperator.NOT_LIKE.equals(op)
|| ComparisonOperator.CONTAINS.equals(op)
|| ComparisonOperator.NOT_CONTAINS.equals(op)
|| ComparisonOperator.IN.equals(op) || ComparisonOperator.IS.equals(op)
|| ComparisonOperator.IS_NOT.equals(op)
|| ComparisonOperator.NOT_EQUAL.equals(op)
|| ComparisonOperator.EQUAL.equals(op)
|| ComparisonOperator.NOT_IN.equals(op)) {
qualifierFilters.add(entry);
} else {
LOG.warn("Don't support operation: \"" + op + "\" on tag field: " + fieldName
+ " yet, going to ignore");
throw new IllegalArgumentException("Don't support operation: " + op
+ " on tag field: " + fieldName
+ ", avaliable options: =, =!, =~, !=~, in, not in, contains, not contains");
}
} else {
qualifierFilters.add(entry);
}
}
// Build RowFilter for equal tags
list.addFilter(buildTagFilter(tagFilters));
// Build SingleColumnValueFilter
FilterList qualifierFilterList = buildQualifierFilter(qualifierFilters);
if (qualifierFilterList != null && qualifierFilterList.getFilters().size() > 0) {
list.addFilter(qualifierFilterList);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Ignore empty qualifier filter from " + qualifierFilters.toString());
}
}
fltList.addFilter(list);
}
LOG.info("Query: " + orExpr.toString() + " => Filter: " + fltList.toString());
return fltList;
}
Example 13
| Source File: HBaseFilterBuilder.java From eagle with Apache License 2.0 | 4 votes |
|
/**
* all qualifiers' condition must be satisfied.
* <H1>Use RegexStringComparator for:</H1> IN LIKE NOT_LIKE
* <H1>Use SubstringComparator for:</H1> CONTAINS
* <H1>Use EntityQualifierHelper for:</H1> EQUALS NOT_EUQALS LESS LESS_OR_EQUAL GREATER GREATER_OR_EQUAL
* <H2>TODO: Compare performance of RegexStringComparator ,SubstringComparator ,EntityQualifierHelper</H2>
*
* @param qualifierFilters
* @return
*/
protected FilterList buildQualifierFilter(List<QualifierFilterEntity> qualifierFilters) {
FilterList list = new FilterList(Operator.MUST_PASS_ALL);
// iterate all the qualifiers
for (QualifierFilterEntity entry : qualifierFilters) {
// if contains expression based filter
if (entry.getKeyType() == TokenType.EXP || entry.getValueType() == TokenType.EXP
|| entry.getKeyType() != TokenType.ID) {
if (!EagleConfigFactory.load().isCoprocessorEnabled()) {
LOG.warn("Expression in filter may not support, because custom filter and coprocessor is disabled: "
+ entry.toString());
}
list.addFilter(buildExpressionBasedFilter(entry));
continue;
}
// else using SingleColumnValueFilter
String qualifierName = entry.getKey();
if (!isTag(entry.getKey())) {
Qualifier qualifier = ed.getDisplayNameMap().get(entry.getKey());
qualifierName = qualifier.getQualifierName();
}
// Comparator to be used for building HBase Filter
// WritableByteArrayComparable comparator;
ByteArrayComparable comparable;
if (ComparisonOperator.IN.equals(entry.getOp())
|| ComparisonOperator.NOT_IN.equals(entry.getOp())) {
Filter setFilter = buildListQualifierFilter(entry);
if (setFilter != null) {
list.addFilter(setFilter);
}
} else {
// If [=,!=,is,is not] NULL, use NullComparator else throw exception
if (TokenType.NULL.equals(entry.getValueType())) {
if (ComparisonOperator.EQUAL.equals(entry.getOp())
|| ComparisonOperator.NOT_EQUAL.equals(entry.getOp())
|| ComparisonOperator.IS.equals(entry.getOp())
|| ComparisonOperator.IS_NOT.equals(entry.getOp())) {
comparable = new NullComparator();
} else {
throw new IllegalArgumentException("Operation: " + entry.getOp()
+ " with NULL is not supported yet: "
+ entry.toString()
+ ", avaliable options: [=, !=, is, is not] null|NULL");
}
} else if (ComparisonOperator.CONTAINS.equals(entry.getOp())
|| ComparisonOperator.NOT_CONTAINS.equals(entry.getOp())) {
// If [contains, not contains],use SubstringComparator
comparable = new SubstringComparator(entry.getValue());
} else if (ComparisonOperator.LIKE.equals(entry.getOp())
|| ComparisonOperator.NOT_LIKE.equals(entry.getOp())) {
// If [like, not like], use RegexStringComparator
// Use RegexStringComparator for LIKE / NOT_LIKE
RegexStringComparator _comparator = new RegexStringComparator(buildQualifierRegex(entry
.getValue()));
_comparator.setCharset(charset);
comparable = _comparator;
} else {
Class type = EntityQualifierUtils.getType(ed, entry.getKey());
// if type is null (is Tag or not found) or not defined for TypedByteArrayComparator
if (!EagleConfigFactory.load().isCoprocessorEnabled() || type == null
|| TypedByteArrayComparator.get(type) == null) {
comparable = new BinaryComparator(EntityQualifierUtils.toBytes(ed, entry.getKey(),
entry.getValue()));
} else {
comparable = new TypedByteArrayComparator(EntityQualifierUtils
.toBytes(ed, entry.getKey(), entry.getValue()), type);
}
}
SingleColumnValueFilter filter = new SingleColumnValueFilter(ed.getColumnFamily()
.getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()),
comparable);
filter.setFilterIfMissing(filterIfMissing);
list.addFilter(filter);
}
}
return list;
}
Example 14
| Source File: HBaseFilterBuilder.java From eagle with Apache License 2.0 | 4 votes |
|
/**
* Currently use BinaryComparator only
* <h2>TODO:</h2> Possibility to tune performance by using: OR[BinaryComparator,...] instead of
* RegexStringComparator? <br/>
* <br/>
* ! Check op must be IN or NOTIN in caller
*
* @param entry
* @return
*/
private Filter buildListQualifierFilter(QualifierFilterEntity entry) {
List<String> valueSet = EntityQualifierUtils.parseList(entry.getValue());
Iterator<String> it = valueSet.iterator();
String fieldName = entry.getKey();
String qualifierName = fieldName;
if (!ed.isTag(entry.getKey())) {
qualifierName = ed.getDisplayNameMap().get(entry.getKey()).getQualifierName();
}
// TODO: Try to use RegExp just work if possible
// Because single SingleColumnValueFilter is much faster than multi SingleColumnValueFilters in OR
// list.
// Class qualifierType = EntityQualifierHelper.getType(ed,fieldName);
// if( qualifierType == null || qualifierType == String.class){
// boolean first = true;
// StringBuilder filterRegex = new StringBuilder();
// filterRegex.append("^(");
// while(it.hasNext()) {
// String value = it.next();
// if(value == null) {
// logger.warn("ignore empty value in set qualifier filter: "+entry.toString());
// continue;
// }
// if(!first) filterRegex.append("|");
// filterRegex.append(value);
// first = false;
// }
// filterRegex.append(")$");
// RegexStringComparator regexStringComparator = new RegexStringComparator(filterRegex.toString());
// return new SingleColumnValueFilter(ed.getColumnFamily().getBytes(), qualifierName.getBytes(),
// convertToHBaseCompareOp(entry.getOp()), regexStringComparator);
// }else{
FilterList setFilterList;
if (ComparisonOperator.IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ONE);
} else if (ComparisonOperator.NOT_IN.equals(entry.getOp())) {
setFilterList = new FilterList(Operator.MUST_PASS_ALL);
} else {
throw new IllegalArgumentException(String
.format("Don't support operation: %s on LIST type of value yet: %s, valid options: IN/NOT IN [LIST]",
entry.getOp(), entry.toString()));
}
while (it.hasNext()) {
String value = it.next();
BinaryComparator comparator = new BinaryComparator(EntityQualifierUtils.toBytes(ed, fieldName,
value));
SingleColumnValueFilter filter = new SingleColumnValueFilter(ed.getColumnFamily()
.getBytes(), qualifierName.getBytes(), convertToHBaseCompareOp(entry.getOp()), comparator);
filter.setFilterIfMissing(filterIfMissing);
setFilterList.addFilter(filter);
}
return setFilterList;
// }
}
