/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.hbase.filter;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellComparator;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.CompareOperator;
import org.apache.hadoop.hbase.HBaseClassTestRule;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.testclassification.FilterTests;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.wal.WAL;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.ClassRule;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.base.Throwables;
/**
* Test filters at the HRegion doorstep.
*/
@Category({FilterTests.class, MediumTests.class})
public class TestFilter {
@ClassRule
public static final HBaseClassTestRule CLASS_RULE =
HBaseClassTestRule.forClass(TestFilter.class);
private final static Logger LOG = LoggerFactory.getLogger(TestFilter.class);
private HRegion region;
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
@Rule
public TestName name = new TestName();
//
// Rows, Qualifiers, and Values are in two groups, One and Two.
//
private static final byte [][] ROWS_ONE = {
Bytes.toBytes("testRowOne-0"), Bytes.toBytes("testRowOne-1"),
Bytes.toBytes("testRowOne-2"), Bytes.toBytes("testRowOne-3")
};
private static final byte [][] ROWS_TWO = {
Bytes.toBytes("testRowTwo-0"), Bytes.toBytes("testRowTwo-1"),
Bytes.toBytes("testRowTwo-2"), Bytes.toBytes("testRowTwo-3")
};
private static final byte [][] ROWS_THREE = {
Bytes.toBytes("testRowThree-0"), Bytes.toBytes("testRowThree-1"),
Bytes.toBytes("testRowThree-2"), Bytes.toBytes("testRowThree-3")
};
private static final byte [][] ROWS_FOUR = {
Bytes.toBytes("testRowFour-0"), Bytes.toBytes("testRowFour-1"),
Bytes.toBytes("testRowFour-2"), Bytes.toBytes("testRowFour-3")
};
private static final byte [][] FAMILIES = {
Bytes.toBytes("testFamilyOne"), Bytes.toBytes("testFamilyTwo")
};
private static final byte [][] FAMILIES_1 = {
Bytes.toBytes("testFamilyThree"), Bytes.toBytes("testFamilyFour")
};
private static final byte [][] QUALIFIERS_ONE = {
Bytes.toBytes("testQualifierOne-0"), Bytes.toBytes("testQualifierOne-1"),
Bytes.toBytes("testQualifierOne-2"), Bytes.toBytes("testQualifierOne-3")
};
private static final byte [][] QUALIFIERS_TWO = {
Bytes.toBytes("testQualifierTwo-0"), Bytes.toBytes("testQualifierTwo-1"),
Bytes.toBytes("testQualifierTwo-2"), Bytes.toBytes("testQualifierTwo-3")
};
private static final byte [][] QUALIFIERS_THREE = {
Bytes.toBytes("testQualifierThree-0"), Bytes.toBytes("testQualifierThree-1"),
Bytes.toBytes("testQualifierThree-2"), Bytes.toBytes("testQualifierThree-3")
};
private static final byte [][] QUALIFIERS_FOUR = {
Bytes.toBytes("testQualifierFour-0"), Bytes.toBytes("testQualifierFour-1"),
Bytes.toBytes("testQualifierFour-2"), Bytes.toBytes("testQualifierFour-3")
};
private static final byte [][] QUALIFIERS_FIVE = {
Bytes.toBytes("testQualifierFive-0"), Bytes.toBytes("testQualifierFive-1")
};
private static final byte [][] VALUES = {
Bytes.toBytes("testValueOne"), Bytes.toBytes("testValueTwo")
};
byte [][] NEW_FAMILIES = {
Bytes.toBytes("f1"), Bytes.toBytes("f2")
};
private long numRows = (long) ROWS_ONE.length + ROWS_TWO.length;
private long colsPerRow = (long) FAMILIES.length * QUALIFIERS_ONE.length;
@Before
public void setUp() throws Exception {
TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf("TestFilter"));
ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor family0 =
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES[0])
.setVersions(100, 100);
tableDescriptor.setColumnFamily(family0);
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES[1]));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES_1[0]));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES_1[1]));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(NEW_FAMILIES[0]));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(NEW_FAMILIES[1]));
RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
this.region = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
TEST_UTIL.getConfiguration(), tableDescriptor);
// Insert first half
for(byte [] ROW : ROWS_ONE) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for(byte [] QUALIFIER : QUALIFIERS_ONE) {
p.addColumn(FAMILIES[0], QUALIFIER, VALUES[0]);
}
this.region.put(p);
}
for(byte [] ROW : ROWS_TWO) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for(byte [] QUALIFIER : QUALIFIERS_TWO) {
p.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
}
this.region.put(p);
}
// Flush
this.region.flush(true);
// Insert second half (reverse families)
for(byte [] ROW : ROWS_ONE) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for(byte [] QUALIFIER : QUALIFIERS_ONE) {
p.addColumn(FAMILIES[1], QUALIFIER, VALUES[0]);
}
this.region.put(p);
}
for(byte [] ROW : ROWS_TWO) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for(byte [] QUALIFIER : QUALIFIERS_TWO) {
p.addColumn(FAMILIES[0], QUALIFIER, VALUES[1]);
}
this.region.put(p);
}
// Delete the second qualifier from all rows and families
for(byte [] ROW : ROWS_ONE) {
Delete d = new Delete(ROW);
d.addColumns(FAMILIES[0], QUALIFIERS_ONE[1]);
d.addColumns(FAMILIES[1], QUALIFIERS_ONE[1]);
this.region.delete(d);
}
for(byte [] ROW : ROWS_TWO) {
Delete d = new Delete(ROW);
d.addColumns(FAMILIES[0], QUALIFIERS_TWO[1]);
d.addColumns(FAMILIES[1], QUALIFIERS_TWO[1]);
this.region.delete(d);
}
colsPerRow -= 2;
// Delete the second rows from both groups, one column at a time
for(byte [] QUALIFIER : QUALIFIERS_ONE) {
Delete d = new Delete(ROWS_ONE[1]);
d.addColumns(FAMILIES[0], QUALIFIER);
d.addColumns(FAMILIES[1], QUALIFIER);
this.region.delete(d);
}
for(byte [] QUALIFIER : QUALIFIERS_TWO) {
Delete d = new Delete(ROWS_TWO[1]);
d.addColumns(FAMILIES[0], QUALIFIER);
d.addColumns(FAMILIES[1], QUALIFIER);
this.region.delete(d);
}
numRows -= 2;
}
@After
public void tearDown() throws Exception {
HBaseTestingUtility.closeRegionAndWAL(region);
}
@Test
public void testRegionScannerReseek() throws Exception {
// create new rows and column family to show how reseek works..
for (byte[] ROW : ROWS_THREE) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for (byte[] QUALIFIER : QUALIFIERS_THREE) {
p.addColumn(FAMILIES[0], QUALIFIER, VALUES[0]);
}
this.region.put(p);
}
for (byte[] ROW : ROWS_FOUR) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for (byte[] QUALIFIER : QUALIFIERS_FOUR) {
p.addColumn(FAMILIES[1], QUALIFIER, VALUES[1]);
}
this.region.put(p);
}
// Flush
this.region.flush(true);
// Insert second half (reverse families)
for (byte[] ROW : ROWS_THREE) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for (byte[] QUALIFIER : QUALIFIERS_THREE) {
p.addColumn(FAMILIES[1], QUALIFIER, VALUES[0]);
}
this.region.put(p);
}
for (byte[] ROW : ROWS_FOUR) {
Put p = new Put(ROW);
p.setDurability(Durability.SKIP_WAL);
for (byte[] QUALIFIER : QUALIFIERS_FOUR) {
p.addColumn(FAMILIES[0], QUALIFIER, VALUES[1]);
}
this.region.put(p);
}
Scan s = new Scan();
// set a start row
s.withStartRow(ROWS_FOUR[1]);
RegionScanner scanner = region.getScanner(s);
// reseek to row three.
scanner.reseek(ROWS_THREE[1]);
List<Cell> results = new ArrayList<>();
// the results should belong to ROWS_THREE[1]
scanner.next(results);
for (Cell keyValue : results) {
assertTrue("The rows with ROWS_TWO as row key should be appearing.",
CellUtil.matchingRows(keyValue, ROWS_THREE[1]));
}
// again try to reseek to a value before ROWS_THREE[1]
scanner.reseek(ROWS_ONE[1]);
results = new ArrayList<>();
// This time no seek would have been done to ROWS_ONE[1]
scanner.next(results);
for (Cell keyValue : results) {
assertFalse("Cannot rewind back to a value less than previous reseek.",
Bytes.toString(CellUtil.cloneRow(keyValue)).contains("testRowOne"));
}
}
@Test
public void testNoFilter() throws Exception {
// No filter
long expectedRows = this.numRows;
long expectedKeys = this.colsPerRow;
// Both families
Scan s = new Scan();
verifyScan(s, expectedRows, expectedKeys);
// One family
s = new Scan();
s.addFamily(FAMILIES[0]);
verifyScan(s, expectedRows, expectedKeys/2);
}
@Test
public void testPrefixFilter() throws Exception {
// Grab rows from group one (half of total)
long expectedRows = this.numRows / 2;
long expectedKeys = this.colsPerRow;
Scan s = new Scan();
s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne")));
verifyScan(s, expectedRows, expectedKeys);
}
@Test
public void testPrefixFilterWithReverseScan() throws Exception {
// Grab rows from group one (half of total)
long expectedRows = this.numRows / 2;
long expectedKeys = this.colsPerRow;
Scan s = new Scan();
s.setReversed(true);
s.setFilter(new PrefixFilter(Bytes.toBytes("testRowOne")));
verifyScan(s, expectedRows, expectedKeys);
}
@Test
public void testPageFilter() throws Exception {
// KVs in first 6 rows
KeyValue [] expectedKVs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1])
};
// Grab all 6 rows
long expectedRows = 6;
long expectedKeys = this.colsPerRow;
Scan s = new Scan();
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
s.setFilter(new PageFilter(expectedRows));
verifyScanFull(s, expectedKVs);
// Grab first 4 rows (6 cols per row)
expectedRows = 4;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
s.setFilter(new PageFilter(expectedRows));
verifyScanFull(s, Arrays.copyOf(expectedKVs, 24));
// Grab first 2 rows
expectedRows = 2;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
s.setFilter(new PageFilter(expectedRows));
verifyScanFull(s, Arrays.copyOf(expectedKVs, 12));
// Grab first row
expectedRows = 1;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
s.setFilter(new PageFilter(expectedRows));
verifyScanFull(s, Arrays.copyOf(expectedKVs, 6));
}
@Test
public void testPageFilterWithReverseScan() throws Exception {
// KVs in first 6 rows
KeyValue[] expectedKVs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]) };
// Grab all 6 rows
long expectedRows = 6;
long expectedKeys = this.colsPerRow;
Scan s = new Scan();
s.setReversed(true);
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
// Grab first 4 rows (6 cols per row)
expectedRows = 4;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setReversed(true);
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
// Grab first 2 rows
expectedRows = 2;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setReversed(true);
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
// Grab first row
expectedRows = 1;
expectedKeys = this.colsPerRow;
s = new Scan();
s.setReversed(true);
s.setFilter(new PageFilter(expectedRows));
verifyScan(s, expectedRows, expectedKeys);
}
@Test
public void testWhileMatchFilterWithFilterRowWithReverseScan()
throws Exception {
final int pageSize = 4;
Scan s = new Scan();
s.setReversed(true);
WhileMatchFilter filter = new WhileMatchFilter(new PageFilter(pageSize));
s.setFilter(filter);
InternalScanner scanner = this.region.getScanner(s);
int scannerCounter = 0;
while (true) {
boolean isMoreResults = scanner.next(new ArrayList<>());
scannerCounter++;
if (scannerCounter >= pageSize) {
Assert.assertTrue(
"The WhileMatchFilter should now filter all remaining",
filter.filterAllRemaining());
}
if (!isMoreResults) {
break;
}
}
scanner.close();
Assert.assertEquals("The page filter returned more rows than expected",
pageSize, scannerCounter);
}
@Test
public void testWhileMatchFilterWithFilterRowKeyWithReverseScan()
throws Exception {
Scan s = new Scan();
String prefix = "testRowOne";
WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(
Bytes.toBytes(prefix)));
s.setFilter(filter);
s.setReversed(true);
InternalScanner scanner = this.region.getScanner(s);
while (true) {
ArrayList<Cell> values = new ArrayList<>();
boolean isMoreResults = scanner.next(values);
if (!isMoreResults
|| !Bytes.toString(CellUtil.cloneRow(values.get(0))).startsWith(prefix)) {
Assert.assertTrue(
"The WhileMatchFilter should now filter all remaining",
filter.filterAllRemaining());
}
if (!isMoreResults) {
break;
}
}
scanner.close();
}
/**
* Tests the the {@link WhileMatchFilter} works in combination with a
* {@link Filter} that uses the
* {@link Filter#filterRow()} method.
*
* See HBASE-2258.
*
* @throws Exception
*/
@Test
public void testWhileMatchFilterWithFilterRow() throws Exception {
final int pageSize = 4;
Scan s = new Scan();
WhileMatchFilter filter = new WhileMatchFilter(new PageFilter(pageSize));
s.setFilter(filter);
InternalScanner scanner = this.region.getScanner(s);
int scannerCounter = 0;
while (true) {
boolean isMoreResults = scanner.next(new ArrayList<>());
scannerCounter++;
if (scannerCounter >= pageSize) {
assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
}
if (!isMoreResults) {
break;
}
}
assertEquals("The page filter returned more rows than expected", pageSize, scannerCounter);
}
/**
* The following filter simulates a pre-0.96 filter where filterRow() is defined while
* hasFilterRow() returns false
*/
static class OldTestFilter extends FilterBase {
@Override
public byte [] toByteArray() {return null;}
@Override
public boolean hasFilterRow() {
return false;
}
@Override
public boolean filterRow() {
// always filter out rows
return true;
}
@Override
public ReturnCode filterCell(final Cell ignored) throws IOException {
return ReturnCode.INCLUDE;
}
}
/**
* The following test is to ensure old(such as hbase0.94) filterRow() can be correctly fired in
* 0.96+ code base.
*
* See HBASE-10366
*
* @throws Exception
*/
@Test
public void test94FilterRowCompatibility() throws Exception {
Scan s = new Scan();
OldTestFilter filter = new OldTestFilter();
s.setFilter(filter);
InternalScanner scanner = this.region.getScanner(s);
ArrayList<Cell> values = new ArrayList<>();
scanner.next(values);
assertTrue("All rows should be filtered out", values.isEmpty());
}
/**
* Tests the the {@link WhileMatchFilter} works in combination with a
* {@link Filter} that uses the
* {@link Filter#filterRowKey(Cell)} method.
*
* See HBASE-2258.
*
* @throws Exception
*/
@Test
public void testWhileMatchFilterWithFilterRowKey() throws Exception {
Scan s = new Scan();
String prefix = "testRowOne";
WhileMatchFilter filter = new WhileMatchFilter(new PrefixFilter(Bytes.toBytes(prefix)));
s.setFilter(filter);
InternalScanner scanner = this.region.getScanner(s);
while (true) {
ArrayList<Cell> values = new ArrayList<>();
boolean isMoreResults = scanner.next(values);
if (!isMoreResults || !Bytes.toString(CellUtil.cloneRow(values.get(0))).startsWith(prefix)) {
assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
}
if (!isMoreResults) {
break;
}
}
}
/**
* Tests the the {@link WhileMatchFilter} works in combination with a
* {@link Filter} that uses the {@link Filter#filterCell(Cell)} method.
*
* See HBASE-2258.
*
* @throws Exception
*/
@Test
public void testWhileMatchFilterWithFilterCell() throws Exception {
Scan s = new Scan();
WhileMatchFilter filter = new WhileMatchFilter(
new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0], CompareOperator.EQUAL, Bytes.toBytes("foo"))
);
s.setFilter(filter);
InternalScanner scanner = this.region.getScanner(s);
while (true) {
ArrayList<Cell> values = new ArrayList<>();
boolean isMoreResults = scanner.next(values);
assertTrue("The WhileMatchFilter should now filter all remaining", filter.filterAllRemaining());
if (!isMoreResults) {
break;
}
}
}
@Test
public void testInclusiveStopFilter() throws IOException {
// Grab rows from group one
// If we just use start/stop row, we get total/2 - 1 rows
long expectedRows = (this.numRows / 2) - 1;
long expectedKeys = this.colsPerRow;
Scan s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0"))
.withStopRow(Bytes.toBytes("testRowOne-3"));
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = this.numRows / 2;
s = new Scan().withStartRow(Bytes.toBytes("testRowOne-0"));
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-3")));
verifyScan(s, expectedRows, expectedKeys);
// Grab rows from group two
// If we just use start/stop row, we get total/2 - 1 rows
expectedRows = (this.numRows / 2) - 1;
expectedKeys = this.colsPerRow;
s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0"))
.withStopRow(Bytes.toBytes("testRowTwo-3"));
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = this.numRows / 2;
s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-0"));
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-3")));
verifyScan(s, expectedRows, expectedKeys);
}
@Test
public void testInclusiveStopFilterWithReverseScan() throws IOException {
// Grab rows from group one
// If we just use start/stop row, we get total/2 - 1 rows
long expectedRows = (this.numRows / 2) - 1;
long expectedKeys = this.colsPerRow;
Scan s = new Scan().withStartRow(Bytes.toBytes("testRowOne-3"))
.withStopRow(Bytes.toBytes("testRowOne-0"));
s.setReversed(true);
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = this.numRows / 2;
s = new Scan().withStartRow(Bytes.toBytes("testRowOne-3"));
s.setReversed(true);
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowOne-0")));
verifyScan(s, expectedRows, expectedKeys);
// Grab rows from group two
// If we just use start/stop row, we get total/2 - 1 rows
expectedRows = (this.numRows / 2) - 1;
expectedKeys = this.colsPerRow;
s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-3"))
.withStopRow(Bytes.toBytes("testRowTwo-0"));
s.setReversed(true);
verifyScan(s, expectedRows, expectedKeys);
// Now use start row with inclusive stop filter
expectedRows = this.numRows / 2;
s = new Scan().withStartRow(Bytes.toBytes("testRowTwo-3"));
s.setReversed(true);
s.setFilter(new InclusiveStopFilter(Bytes.toBytes("testRowTwo-0")));
verifyScan(s, expectedRows, expectedKeys);
}
@Test
public void testQualifierFilter() throws IOException {
// Match two keys (one from each family) in half the rows
long expectedRows = this.numRows / 2;
long expectedKeys = 2;
Filter f = new QualifierFilter(CompareOperator.EQUAL,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
Scan s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys less than same qualifier
// Expect only two keys (one from each family) in half the rows
expectedRows = this.numRows / 2;
expectedKeys = 2;
f = new QualifierFilter(CompareOperator.LESS,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys less than or equal
// Expect four keys (two from each family) in half the rows
expectedRows = this.numRows / 2;
expectedKeys = 4;
f = new QualifierFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys not equal
// Expect four keys (two from each family)
// Only look in first group of rows
expectedRows = this.numRows / 2;
expectedKeys = 4;
f = new QualifierFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys greater or equal
// Expect four keys (two from each family)
// Only look in first group of rows
expectedRows = this.numRows / 2;
expectedKeys = 4;
f = new QualifierFilter(CompareOperator.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys greater
// Expect two keys (one from each family)
// Only look in first group of rows
expectedRows = this.numRows / 2;
expectedKeys = 2;
f = new QualifierFilter(CompareOperator.GREATER,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys not equal to
// Look across rows and fully validate the keys and ordering
// Expect varied numbers of keys, 4 per row in group one, 6 per row in group two
f = new QualifierFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(QUALIFIERS_ONE[2]));
s = new Scan();
s.setFilter(f);
KeyValue [] kvs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
// Test across rows and groups with a regex
// Filter out "test*-2"
// Expect 4 keys per row across both groups
f = new QualifierFilter(CompareOperator.NOT_EQUAL,
new RegexStringComparator("test.+-2"));
s = new Scan();
s.setFilter(f);
kvs = new KeyValue [] {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
}
@Test
public void testFamilyFilter() throws IOException {
// Match family, only half of columns returned.
long expectedRows = this.numRows;
long expectedKeys = this.colsPerRow / 2;
Filter f = new FamilyFilter(CompareOperator.EQUAL,
new BinaryComparator(Bytes.toBytes("testFamilyOne")));
Scan s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys less than given family, should return nothing
expectedRows = 0;
expectedKeys = 0;
f = new FamilyFilter(CompareOperator.LESS,
new BinaryComparator(Bytes.toBytes("testFamily")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys less than or equal, should return half of columns
expectedRows = this.numRows;
expectedKeys = this.colsPerRow / 2;
f = new FamilyFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testFamilyOne")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys from second family
// look only in second group of rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow / 2;
f = new FamilyFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testFamilyOne")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match all columns
// look only in second group of rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new FamilyFilter(CompareOperator.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testFamilyOne")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match all columns in second family
// look only in second group of rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow / 2;
f = new FamilyFilter(CompareOperator.GREATER,
new BinaryComparator(Bytes.toBytes("testFamilyOne")));
s = new Scan().withStartRow(HConstants.EMPTY_START_ROW)
.withStopRow(Bytes.toBytes("testRowTwo"));
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys not equal to given family
// Look across rows and fully validate the keys and ordering
f = new FamilyFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(FAMILIES[1]));
s = new Scan();
s.setFilter(f);
KeyValue [] kvs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
// Test across rows and groups with a regex
// Filter out "test*-2"
// Expect 4 keys per row across both groups
f = new FamilyFilter(CompareOperator.NOT_EQUAL,
new RegexStringComparator("test.*One"));
s = new Scan();
s.setFilter(f);
kvs = new KeyValue [] {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
}
@Test
public void testRowFilter() throws IOException {
// Match a single row, all keys
long expectedRows = 1;
long expectedKeys = this.colsPerRow;
Filter f = new RowFilter(CompareOperator.EQUAL,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
Scan s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match a two rows, one from each group, using regex
expectedRows = 2;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.EQUAL,
new RegexStringComparator("testRow.+-2"));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match rows less than
// Expect all keys in one row
expectedRows = 1;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.LESS,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match rows less than or equal
// Expect all keys in two rows
expectedRows = 2;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match rows not equal
// Expect all keys in all but one row
expectedRows = this.numRows - 1;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys greater or equal
// Expect all keys in all but one row
expectedRows = this.numRows - 1;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match keys greater
// Expect all keys in all but two rows
expectedRows = this.numRows - 2;
expectedKeys = this.colsPerRow;
f = new RowFilter(CompareOperator.GREATER,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match rows not equal to testRowTwo-2
// Look across rows and fully validate the keys and ordering
// Should see all keys in all rows but testRowTwo-2
f = new RowFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testRowOne-2")));
s = new Scan();
s.setFilter(f);
KeyValue [] kvs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
// Test across rows and groups with a regex
// Filter out everything that doesn't match "*-2"
// Expect all keys in two rows
f = new RowFilter(CompareOperator.EQUAL,
new RegexStringComparator(".+-2"));
s = new Scan();
s.setFilter(f);
kvs = new KeyValue [] {
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1])
};
verifyScanFull(s, kvs);
}
@Test
public void testValueFilter() throws IOException {
// Match group one rows
long expectedRows = this.numRows / 2;
long expectedKeys = this.colsPerRow;
Filter f = new ValueFilter(CompareOperator.EQUAL,
new BinaryComparator(Bytes.toBytes("testValueOne")));
Scan s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match group two rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.EQUAL,
new BinaryComparator(Bytes.toBytes("testValueTwo")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match all values using regex
expectedRows = this.numRows;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.EQUAL,
new RegexStringComparator("testValue((One)|(Two))"));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values less than
// Expect group one rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.LESS,
new BinaryComparator(Bytes.toBytes("testValueTwo")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values less than or equal
// Expect all rows
expectedRows = this.numRows;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testValueTwo")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values less than or equal
// Expect group one rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testValueOne")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values not equal
// Expect half the rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testValueOne")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values greater or equal
// Expect all rows
expectedRows = this.numRows;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.GREATER_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("testValueOne")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values greater
// Expect half rows
expectedRows = this.numRows / 2;
expectedKeys = this.colsPerRow;
f = new ValueFilter(CompareOperator.GREATER,
new BinaryComparator(Bytes.toBytes("testValueOne")));
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, expectedRows, expectedKeys);
// Match values not equal to testValueOne
// Look across rows and fully validate the keys and ordering
// Should see all keys in all group two rows
f = new ValueFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testValueOne")));
s = new Scan();
s.setFilter(f);
KeyValue [] kvs = {
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
}
@Test
public void testSkipFilter() throws IOException {
// Test for qualifier regex: "testQualifierOne-2"
// Should only get rows from second group, and all keys
Filter f = new SkipFilter(new QualifierFilter(CompareOperator.NOT_EQUAL,
new BinaryComparator(Bytes.toBytes("testQualifierOne-2"))));
Scan s = new Scan();
s.setFilter(f);
KeyValue [] kvs = {
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanFull(s, kvs);
}
// TODO: This is important... need many more tests for ordering, etc
// There are limited tests elsewhere but we need HRegion level ones here
@Test
public void testFilterList() throws IOException {
// Test getting a single row, single key using Row, Qualifier, and Value
// regular expression and substring filters
// Use must pass all
List<Filter> filters = new ArrayList<>();
filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
Scan s = new Scan();
s.addFamily(FAMILIES[0]);
s.setFilter(f);
KeyValue [] kvs = {
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0])
};
verifyScanFull(s, kvs);
// Test getting everything with a MUST_PASS_ONE filter including row, qf, val
// regular expression and substring filters
filters.clear();
filters.add(new RowFilter(CompareOperator.EQUAL, new RegexStringComparator(".+Two.+")));
filters.add(new QualifierFilter(CompareOperator.EQUAL, new RegexStringComparator(".+-2")));
filters.add(new ValueFilter(CompareOperator.EQUAL, new SubstringComparator("One")));
f = new FilterList(Operator.MUST_PASS_ONE, filters);
s = new Scan();
s.setFilter(f);
verifyScanNoEarlyOut(s, this.numRows, this.colsPerRow);
}
@Test
public void testFirstKeyOnlyFilter() throws IOException {
Scan s = new Scan();
s.setFilter(new FirstKeyOnlyFilter());
// Expected KVs, the first KV from each of the remaining 6 rows
KeyValue [] kvs = {
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1])
};
verifyScanFull(s, kvs);
}
@Test
public void testFilterListWithSingleColumnValueFilter() throws IOException {
// Test for HBASE-3191
// Scan using SingleColumnValueFilter
SingleColumnValueFilter f1 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0],
CompareOperator.EQUAL, VALUES[0]);
f1.setFilterIfMissing( true );
Scan s1 = new Scan();
s1.addFamily(FAMILIES[0]);
s1.setFilter(f1);
KeyValue [] kvs1 = {
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
};
verifyScanNoEarlyOut(s1, 3, 3);
verifyScanFull(s1, kvs1);
// Scan using another SingleColumnValueFilter, expect disjoint result
SingleColumnValueFilter f2 = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_TWO[0],
CompareOperator.EQUAL, VALUES[1]);
f2.setFilterIfMissing( true );
Scan s2 = new Scan();
s2.addFamily(FAMILIES[0]);
s2.setFilter(f2);
KeyValue [] kvs2 = {
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanNoEarlyOut(s2, 3, 3);
verifyScanFull(s2, kvs2);
// Scan, ORing the two previous filters, expect unified result
FilterList f = new FilterList(Operator.MUST_PASS_ONE);
f.addFilter(f1);
f.addFilter(f2);
Scan s = new Scan();
s.addFamily(FAMILIES[0]);
s.setFilter(f);
KeyValue [] kvs = {
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
verifyScanNoEarlyOut(s, 6, 3);
verifyScanFull(s, kvs);
}
// HBASE-9747
@Test
public void testFilterListWithPrefixFilter() throws IOException {
byte[] family = Bytes.toBytes("f1");
byte[] qualifier = Bytes.toBytes("q1");
TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
new TableDescriptorBuilder.ModifyableTableDescriptor(
TableName.valueOf(name.getMethodName()));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(family));
RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
HRegion testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
TEST_UTIL.getConfiguration(), tableDescriptor);
for(int i=0; i<5; i++) {
Put p = new Put(Bytes.toBytes((char)('a'+i) + "row"));
p.setDurability(Durability.SKIP_WAL);
p.addColumn(family, qualifier, Bytes.toBytes(String.valueOf(111 + i)));
testRegion.put(p);
}
testRegion.flush(true);
// rows starting with "b"
PrefixFilter pf = new PrefixFilter(new byte[] {'b'}) ;
// rows with value of column 'q1' set to '113'
SingleColumnValueFilter scvf = new SingleColumnValueFilter(
family, qualifier, CompareOperator.EQUAL, Bytes.toBytes("113"));
// combine these two with OR in a FilterList
FilterList filterList = new FilterList(Operator.MUST_PASS_ONE, pf, scvf);
Scan s1 = new Scan();
s1.setFilter(filterList);
InternalScanner scanner = testRegion.getScanner(s1);
List<Cell> results = new ArrayList<>();
int resultCount = 0;
while (scanner.next(results)) {
resultCount++;
byte[] row = CellUtil.cloneRow(results.get(0));
LOG.debug("Found row: " + Bytes.toStringBinary(row));
assertTrue(Bytes.equals(row, Bytes.toBytes("brow"))
|| Bytes.equals(row, Bytes.toBytes("crow")));
results.clear();
}
assertEquals(2, resultCount);
scanner.close();
WAL wal = ((HRegion)testRegion).getWAL();
((HRegion)testRegion).close();
wal.close();
}
@Test
public void testSingleColumnValueFilter() throws IOException {
// From HBASE-1821
// Desired action is to combine two SCVF in a FilterList
// Want to return only rows that match both conditions
// Need to change one of the group one columns to use group two value
Put p = new Put(ROWS_ONE[2]);
p.addColumn(FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]);
this.region.put(p);
// Now let's grab rows that have Q_ONE[0](VALUES[0]) and Q_ONE[2](VALUES[1])
// Since group two rows don't have these qualifiers, they will pass
// so limiting scan to group one
List<Filter> filters = new ArrayList<>();
filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0],
CompareOperator.EQUAL, VALUES[0]));
filters.add(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2],
CompareOperator.EQUAL, VALUES[1]));
Filter f = new FilterList(Operator.MUST_PASS_ALL, filters);
Scan s = new Scan().withStartRow(ROWS_ONE[0]).withStopRow(ROWS_TWO[0]);
s.addFamily(FAMILIES[0]);
s.setFilter(f);
// Expect only one row, all qualifiers
KeyValue [] kvs = {
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0])
};
verifyScanNoEarlyOut(s, 1, 3);
verifyScanFull(s, kvs);
// In order to get expected behavior without limiting to group one
// need to wrap SCVFs in SkipFilters
filters = new ArrayList<>();
filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0],
CompareOperator.EQUAL, VALUES[0])));
filters.add(new SkipFilter(new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[2],
CompareOperator.EQUAL, VALUES[1])));
f = new FilterList(Operator.MUST_PASS_ALL, filters);
s = new Scan().withStartRow(ROWS_ONE[0]).withStopRow(ROWS_TWO[0]);
s.addFamily(FAMILIES[0]);
s.setFilter(f);
// Expect same KVs
verifyScanNoEarlyOut(s, 1, 3);
verifyScanFull(s, kvs);
// More tests from HBASE-1821 for Clint and filterIfMissing flag
byte [][] ROWS_THREE = {
Bytes.toBytes("rowThree-0"), Bytes.toBytes("rowThree-1"),
Bytes.toBytes("rowThree-2"), Bytes.toBytes("rowThree-3")
};
// Give row 0 and 2 QUALIFIERS_ONE[0] (VALUE[0] VALUE[1])
// Give row 1 and 3 QUALIFIERS_ONE[1] (VALUE[0] VALUE[1])
KeyValue [] srcKVs = new KeyValue [] {
new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_THREE[1], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]),
new KeyValue(ROWS_THREE[2], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]),
new KeyValue(ROWS_THREE[3], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[1])
};
for(KeyValue kv : srcKVs) {
Put put = new Put(CellUtil.cloneRow(kv)).add(kv);
put.setDurability(Durability.SKIP_WAL);
this.region.put(put);
}
// Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = false
// Expect 3 rows (0, 2, 3)
SingleColumnValueFilter scvf = new SingleColumnValueFilter(FAMILIES[0],
QUALIFIERS_ONE[0], CompareOperator.EQUAL, VALUES[0]);
s = new Scan().withStartRow(ROWS_THREE[0]).withStopRow(Bytes.toBytes("rowThree-4"));
s.addFamily(FAMILIES[0]);
s.setFilter(scvf);
kvs = new KeyValue [] { srcKVs[0], srcKVs[2], srcKVs[3] };
verifyScanFull(s, kvs);
// Match VALUES[0] against QUALIFIERS_ONE[0] with filterIfMissing = true
// Expect 1 row (0)
scvf = new SingleColumnValueFilter(FAMILIES[0], QUALIFIERS_ONE[0],
CompareOperator.EQUAL, VALUES[0]);
scvf.setFilterIfMissing(true);
s = new Scan().withStartRow(ROWS_THREE[0]).withStopRow(Bytes.toBytes("rowThree-4"));
s.addFamily(FAMILIES[0]);
s.setFilter(scvf);
kvs = new KeyValue [] { srcKVs[0] };
verifyScanFull(s, kvs);
// Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true
// Expect 1 row (3)
scvf = new SingleColumnValueFilter(FAMILIES[0],
QUALIFIERS_ONE[1], CompareOperator.EQUAL, VALUES[1]);
scvf.setFilterIfMissing(true);
s = new Scan().withStartRow(ROWS_THREE[0]).withStopRow(Bytes.toBytes("rowThree-4"));
s.addFamily(FAMILIES[0]);
s.setFilter(scvf);
kvs = new KeyValue [] { srcKVs[3] };
verifyScanFull(s, kvs);
// Add QUALIFIERS_ONE[1] to ROWS_THREE[0] with VALUES[0]
KeyValue kvA = new KeyValue(ROWS_THREE[0], FAMILIES[0], QUALIFIERS_ONE[1], VALUES[0]);
this.region.put(new Put(CellUtil.cloneRow(kvA)).add(kvA));
// Match VALUES[1] against QUALIFIERS_ONE[1] with filterIfMissing = true
// Expect 1 row (3)
scvf = new SingleColumnValueFilter(FAMILIES[0],
QUALIFIERS_ONE[1], CompareOperator.EQUAL, VALUES[1]);
scvf.setFilterIfMissing(true);
s = new Scan().withStartRow(ROWS_THREE[0]).withStopRow(Bytes.toBytes("rowThree-4"));
s.addFamily(FAMILIES[0]);
s.setFilter(scvf);
kvs = new KeyValue [] { srcKVs[3] };
verifyScanFull(s, kvs);
}
@Test
public void testColumnValueFilter() throws Exception {
// Prepare test rows:
for (int i = 0; i < 2; i++) {
for (int j = 0; j < ROWS_ONE.length; j++) {
Put p1 = new Put(ROWS_ONE[j]).setDurability(Durability.SKIP_WAL);
Put p2 = new Put(ROWS_TWO[j]).setDurability(Durability.SKIP_WAL);
for (byte[] q5 : QUALIFIERS_FIVE) {
p1.addColumn(FAMILIES[0], q5, VALUES[0 + i]).addColumn(FAMILIES[1], q5, VALUES[0 + i]);
p2.addColumn(FAMILIES[0], q5, VALUES[1 - i]).addColumn(FAMILIES[1], q5, VALUES[1 - i]);
}
this.region.put(p1);
this.region.put(p2);
}
this.region.flush(true);
}
// 1. Test = f[0]:q5[0]:v[1]
Scan scan = new Scan().setFilter(
new ColumnValueFilter(FAMILIES[0], QUALIFIERS_FIVE[0], CompareOperator.EQUAL, VALUES[1]));
KeyValue[] expectedEquals =
{ new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]) };
verifyScanFull(scan, expectedEquals);
// 2. Test > f[0]:q5[0]:v[0]
scan.setFilter(
new ColumnValueFilter(FAMILIES[0], QUALIFIERS_FIVE[0], CompareOperator.GREATER, VALUES[0]));
KeyValue[] expectedGreater =
{ new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]) };
verifyScanFull(scan, expectedGreater);
// 3. Test >= f[0]:q5[0]:v[0]
// also test readAllVersions(), since FAMILIES[0] allow multiple versions.
scan.readAllVersions().setFilter(new ColumnValueFilter(FAMILIES[0], QUALIFIERS_FIVE[0],
CompareOperator.GREATER_OR_EQUAL, VALUES[0]));
KeyValue[] expectedGreaterOrEqual =
{ new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_ONE[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]) };
verifyScanFull(scan, expectedGreaterOrEqual);
// 4. Test < f[1]:q5[1]:v[1], FAMILIES[1] doesn't support multiple versions
scan.readVersions(1).setFilter(new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1],
CompareOperator.LESS, VALUES[1]));
KeyValue[] expectedLess =
{ new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) };
verifyScanFull(scan, expectedLess);
// 5. Test <= f[1]:q5[0]:v[1]
scan.setFilter(new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1],
CompareOperator.LESS_OR_EQUAL, VALUES[1]));
KeyValue[] expectedLessOrEqual =
{ new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) };
verifyScanFull(scan, expectedLessOrEqual);
// 6. Test != f[1]:q5[1]:v[1]
scan.setFilter(
new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1], CompareOperator.NOT_EQUAL, VALUES[1]));
KeyValue[] expectedNotEqual =
{ new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) };
verifyScanFull(scan, expectedNotEqual);
// 7. Test FilterList(MUST_PASS_ONE) combining ColumnValueFilter and QualifierFilter
// (ColumnValueFilter, != f[1]:q5[1]:v[1]) || (QualifierFilter, = q5[0])
List<Filter> orFilters = new ArrayList<>(2);
orFilters.add(
new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1], CompareOperator.NOT_EQUAL, VALUES[1]));
orFilters.add(
new QualifierFilter(CompareOperator.EQUAL, new BinaryComparator(QUALIFIERS_FIVE[0])));
scan.setFilter(new FilterList(Operator.MUST_PASS_ONE, orFilters));
KeyValue[] expectedMustPassOne =
{ new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[1], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]), // this pass scvf
new KeyValue(ROWS_TWO[1], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]), // this pass scvf
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]), // this pass scvf
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[0], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) }; // this pass scvf
verifyScanFull(scan, expectedMustPassOne);
// 8. Test FilterList(MUST_PASS_ALL) combining ColumnValueFilter and RowFilter
// (ColumnValueFilter, != f[1]:q5[1]:v[1]) && (RowFilter, = prefix:"testRow")
List<Filter> andFilters = new ArrayList<>(2);
andFilters.add(
new ColumnValueFilter(FAMILIES[1], QUALIFIERS_FIVE[1], CompareOperator.NOT_EQUAL, VALUES[1]));
andFilters.add(new RowFilter(CompareOperator.EQUAL,
new BinaryPrefixComparator(Bytes.toBytes("testRow"))));
scan.setFilter(new FilterList(Operator.MUST_PASS_ALL, andFilters));
KeyValue[] expectedMustPassAll =
{ new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) };
verifyScanFull(scan, expectedMustPassAll);
// 9. Test specified columns with FilterList(MUST_PASS_ONE) which sused in case 7.
// Result is different from case 7, because column is strongly constrained by specified columns
Scan anotherScan = new Scan().addColumn(FAMILIES[1], QUALIFIERS_FIVE[1])
.setFilter(new FilterList(Operator.MUST_PASS_ONE, orFilters));
KeyValue[] expectedValues =
{ new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[1], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_FIVE[1], VALUES[0]) };
verifyScanFull(anotherScan, expectedValues);
}
private void verifyScan(Scan s, long expectedRows, long expectedKeys)
throws IOException {
InternalScanner scanner = this.region.getScanner(s);
List<Cell> results = new ArrayList<>();
int i = 0;
for (boolean done = true; done; i++) {
done = scanner.next(results);
Arrays.sort(results.toArray(new Cell[results.size()]), CellComparator.getInstance());
LOG.info("counter=" + i + ", " + results);
if (results.isEmpty()) break;
assertTrue("Scanned too many rows! Only expected " + expectedRows +
" total but already scanned " + (i+1), expectedRows > i);
assertEquals("Expected " + expectedKeys + " keys per row but " +
"returned " + results.size(), expectedKeys, results.size());
results.clear();
}
assertEquals("Expected " + expectedRows + " rows but scanned " + i +
" rows", expectedRows, i);
}
private void verifyScanNoEarlyOut(Scan s, long expectedRows,
long expectedKeys)
throws IOException {
InternalScanner scanner = this.region.getScanner(s);
List<Cell> results = new ArrayList<>();
int i = 0;
for (boolean done = true; done; i++) {
done = scanner.next(results);
Arrays.sort(results.toArray(new Cell[results.size()]),
CellComparator.getInstance());
LOG.info("counter=" + i + ", " + results);
if(results.isEmpty()) break;
assertTrue("Scanned too many rows! Only expected " + expectedRows +
" total but already scanned " + (i+1), expectedRows > i);
assertEquals("Expected " + expectedKeys + " keys per row but " +
"returned " + results.size(), expectedKeys, results.size());
results.clear();
}
assertEquals("Expected " + expectedRows + " rows but scanned " + i +
" rows", expectedRows, i);
}
private void verifyScanFull(Scan s, KeyValue [] kvs)
throws IOException {
InternalScanner scanner = this.region.getScanner(s);
List<Cell> results = new ArrayList<>();
int row = 0;
int idx = 0;
for (boolean done = true; done; row++) {
done = scanner.next(results);
Arrays.sort(results.toArray(new Cell[results.size()]),
CellComparator.getInstance());
if(results.isEmpty()) break;
assertTrue("Scanned too many keys! Only expected " + kvs.length +
" total but already scanned " + (results.size() + idx) +
(results.isEmpty() ? "" : "(" + results.get(0).toString() + ")"),
kvs.length >= idx + results.size());
for (Cell kv : results) {
LOG.info("row=" + row + ", result=" + kv.toString() +
", match=" + kvs[idx].toString());
assertTrue("Row mismatch", CellUtil.matchingRows(kv, kvs[idx]));
assertTrue("Family mismatch", CellUtil.matchingFamily(kv, kvs[idx]));
assertTrue("Qualifier mismatch", CellUtil.matchingQualifier(kv, kvs[idx]));
assertTrue("Value mismatch", CellUtil.matchingValue(kv, kvs[idx]));
idx++;
}
results.clear();
}
LOG.info("Looked at " + row + " rows with " + idx + " keys");
assertEquals("Expected " + kvs.length + " total keys but scanned " + idx,
kvs.length, idx);
}
private void verifyScanFullNoValues(Scan s, KeyValue [] kvs, boolean useLen)
throws IOException {
InternalScanner scanner = this.region.getScanner(s);
List<Cell> results = new ArrayList<>();
int row = 0;
int idx = 0;
for (boolean more = true; more; row++) {
more = scanner.next(results);
Arrays.sort(results.toArray(new Cell[results.size()]),
CellComparator.getInstance());
if(results.isEmpty()) break;
assertTrue("Scanned too many keys! Only expected " + kvs.length +
" total but already scanned " + (results.size() + idx) +
(results.isEmpty() ? "" : "(" + results.get(0).toString() + ")"),
kvs.length >= idx + results.size());
for(Cell kv : results) {
LOG.info("row=" + row + ", result=" + kv.toString() +
", match=" + kvs[idx].toString());
assertTrue("Row mismatch", CellUtil.matchingRows(kv, kvs[idx]));
assertTrue("Family mismatch", CellUtil.matchingFamily(kv, kvs[idx]));
assertTrue("Qualifier mismatch", CellUtil.matchingQualifier(kv, kvs[idx]));
assertFalse("Should not have returned whole value", CellUtil.matchingValue(kv, kvs[idx]));
if (useLen) {
assertEquals("Value in result is not SIZEOF_INT", Bytes.SIZEOF_INT, kv.getValueLength());
LOG.info("idx = " + idx + ", len=" + kvs[idx].getValueLength()
+ ", actual=" + Bytes.toInt(CellUtil.cloneValue(kv)));
assertEquals("Scan value should be the length of the actual value. ",
kvs[idx].getValueLength(), Bytes.toInt(CellUtil.cloneValue(kv)) );
LOG.info("good");
} else {
assertEquals("Value in result is not empty", 0, kv.getValueLength());
}
idx++;
}
results.clear();
}
LOG.info("Looked at " + row + " rows with " + idx + " keys");
assertEquals("Expected " + kvs.length + " total keys but scanned " + idx,
kvs.length, idx);
}
@Test
public void testColumnPaginationFilterColumnOffset() throws Exception {
KeyValue [] expectedKVs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
};
KeyValue [] expectedKVs1 = {
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1])
};
KeyValue [] expectedKVs2 = {
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1])
};
KeyValue [] expectedKVs3 = {
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
};
Scan s = new Scan();
// Page size 1.
long expectedRows = 6;
long expectedKeys = 1;
s.setFilter(new ColumnPaginationFilter(1, QUALIFIERS_ONE[1]));
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs);
// Page size 2.
expectedRows = 3;
expectedKeys = 2;
s = new Scan();
s.setFilter(new ColumnPaginationFilter(2, QUALIFIERS_TWO[2]));
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs1);
// Page size 3 across multiple column families.
expectedRows = 3;
expectedKeys = 3;
s = new Scan();
s.setFilter(new ColumnPaginationFilter(3, QUALIFIERS_TWO[2]));
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs2);
// Page size 2 restricted to one column family.
expectedRows = 3;
expectedKeys = 2;
s = new Scan();
s.addFamily(FAMILIES[1]);
s.setFilter(new ColumnPaginationFilter(2, QUALIFIERS_TWO[2]));
this.verifyScanFull(s, expectedKVs3);
}
@Test
public void testLatestVersionFilterWithExplicitColumn() throws Exception {
// Add multiple versions
Put p = new Put(ROWS_ONE[0]);
p.setDurability(Durability.SKIP_WAL);
p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]);
this.region.put(p);
p = new Put(ROWS_ONE[0]);
p.setDurability(Durability.SKIP_WAL);
p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]);
this.region.put(p);
this.region.flush(true);
Scan s = new Scan();
s.setFilter(new FilterBase() {
@Override
public ReturnCode filterCell(Cell c) throws IOException {
return ReturnCode.INCLUDE_AND_NEXT_COL;
}
});
s.readVersions(100);
s.addColumn(FAMILIES[0], QUALIFIERS_ONE[0]);
s.addColumn(FAMILIES[0], QUALIFIERS_ONE[1]);
s.addColumn(FAMILIES[0], QUALIFIERS_ONE[2]);
s.addColumn(FAMILIES[0], QUALIFIERS_ONE[3]);
s.addColumn(FAMILIES[0], QUALIFIERS_TWO[0]);
s.addColumn(FAMILIES[0], QUALIFIERS_TWO[1]);
s.addColumn(FAMILIES[0], QUALIFIERS_TWO[2]);
s.addColumn(FAMILIES[0], QUALIFIERS_TWO[3]);
KeyValue[] kvs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[1]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]), };
verifyScanFull(s, kvs);
}
@Test
public void testColumnPaginationFilter() throws Exception {
// Test that the filter skips multiple column versions.
Put p = new Put(ROWS_ONE[0]);
p.setDurability(Durability.SKIP_WAL);
p.addColumn(FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]);
this.region.put(p);
this.region.flush(true);
// Set of KVs (page: 1; pageSize: 1) - the first set of 1 column per row
KeyValue [] expectedKVs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1])
};
// Set of KVs (page: 3; pageSize: 1) - the third set of 1 column per row
KeyValue [] expectedKVs2 = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
};
// Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row
KeyValue [] expectedKVs3 = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
};
// Set of KVs (page: 2; pageSize 2) - the 2nd set of 2 columns per row
KeyValue [] expectedKVs4 = {
};
long expectedRows = this.numRows;
long expectedKeys = 1;
Scan s = new Scan();
// Page 1; 1 Column per page (Limit 1, Offset 0)
s.setFilter(new ColumnPaginationFilter(1,0));
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs);
// Page 3; 1 Result per page (Limit 1, Offset 2)
s.setFilter(new ColumnPaginationFilter(1,2));
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs2);
// Page 2; 2 Results per page (Limit 2, Offset 2)
s.setFilter(new ColumnPaginationFilter(2,2));
expectedKeys = 2;
verifyScan(s, expectedRows, expectedKeys);
this.verifyScanFull(s, expectedKVs3);
// Page 8; 20 Results per page (no results) (Limit 20, Offset 140)
s.setFilter(new ColumnPaginationFilter(20,140));
expectedKeys = 0;
expectedRows = 0;
verifyScan(s, expectedRows, 0);
this.verifyScanFull(s, expectedKVs4);
}
@Test
public void testKeyOnlyFilter() throws Exception {
// KVs in first 6 rows
KeyValue [] expectedKVs = {
// testRowOne-0
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[0], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-2
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[2], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowOne-3
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[0], QUALIFIERS_ONE[3], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[0], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[2], VALUES[0]),
new KeyValue(ROWS_ONE[3], FAMILIES[1], QUALIFIERS_ONE[3], VALUES[0]),
// testRowTwo-0
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[0], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-2
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[2], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1]),
// testRowTwo-3
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[0], QUALIFIERS_TWO[3], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[0], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[2], VALUES[1]),
new KeyValue(ROWS_TWO[3], FAMILIES[1], QUALIFIERS_TWO[3], VALUES[1])
};
// Grab all 6 rows
long expectedRows = 6;
long expectedKeys = this.colsPerRow;
for (boolean useLen : new boolean[]{false,true}) {
Scan s = new Scan();
s.setFilter(new KeyOnlyFilter(useLen));
verifyScan(s, expectedRows, expectedKeys);
verifyScanFullNoValues(s, expectedKVs, useLen);
}
}
/**
* Filter which makes sleeps for a second between each row of a scan.
* This can be useful for manual testing of bugs like HBASE-5973. For example:
* <code>
* create 't1', 'f1'
* 1.upto(100) { |x| put 't1', 'r' + x.to_s, 'f1:q1', 'hi' }
* import org.apache.hadoop.hbase.filter.TestFilter
* scan 't1', { FILTER => TestFilter::SlowScanFilter.new(), CACHE => 50 }
* </code>
*/
public static class SlowScanFilter extends FilterBase {
private static Thread ipcHandlerThread = null;
@Override
public byte [] toByteArray() {return null;}
@Override
public ReturnCode filterCell(final Cell ignored) throws IOException {
return ReturnCode.INCLUDE;
}
@Override
public boolean filterRow() throws IOException {
ipcHandlerThread = Thread.currentThread();
try {
LOG.info("Handler thread " + ipcHandlerThread + " sleeping in filter...");
Thread.sleep(1000);
} catch (InterruptedException e) {
Throwables.propagate(e);
}
return super.filterRow();
}
}
@Test
@Ignore("TODO: intentionally disabled?")
public void testNestedFilterListWithSCVF() throws IOException {
byte[] columnStatus = Bytes.toBytes("S");
TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
new TableDescriptorBuilder.ModifyableTableDescriptor(
TableName.valueOf(name.getMethodName()));
tableDescriptor.setColumnFamily(
new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILIES[0]));
RegionInfo info = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
HRegion testRegion = HBaseTestingUtility.createRegionAndWAL(info, TEST_UTIL.getDataTestDir(),
TEST_UTIL.getConfiguration(), tableDescriptor);
for(int i=0; i<10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.setDurability(Durability.SKIP_WAL);
p.addColumn(FAMILIES[0], columnStatus, Bytes.toBytes(i % 2));
testRegion.put(p);
}
testRegion.flush(true);
// 1. got rows > "row4"
Filter rowFilter = new RowFilter(CompareOperator.GREATER,new BinaryComparator(Bytes.toBytes("row4")));
Scan s1 = new Scan();
s1.setFilter(rowFilter);
InternalScanner scanner = testRegion.getScanner(s1);
List<Cell> results = new ArrayList<>();
int i = 5;
for (boolean done = true; done; i++) {
done = scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
// 2. got rows <= "row4" and S=
FilterList subFilterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
Filter subFilter1 = new RowFilter(CompareOperator.LESS_OR_EQUAL,
new BinaryComparator(Bytes.toBytes("row4")));
subFilterList.addFilter(subFilter1);
Filter subFilter2 = new SingleColumnValueFilter(FAMILIES[0], columnStatus, CompareOperator.EQUAL,
Bytes.toBytes(0));
subFilterList.addFilter(subFilter2);
s1 = new Scan();
s1.setFilter(subFilterList);
scanner = testRegion.getScanner(s1);
results = new ArrayList<>();
for (i=0; i<=4; i+=2) {
scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
assertFalse(scanner.next(results));
// 3. let's begin to verify nested filter list
// 3.1 add rowFilter, then add subFilterList
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
filterList.addFilter(rowFilter);
filterList.addFilter(subFilterList);
s1 = new Scan();
s1.setFilter(filterList);
scanner = testRegion.getScanner(s1);
results = new ArrayList<>();
for (i=0; i<=4; i+=2) {
scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
for (i=5; i<=9; i++) {
scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
assertFalse(scanner.next(results));
// 3.2 MAGIC here! add subFilterList first, then add rowFilter
filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
filterList.addFilter(subFilterList);
filterList.addFilter(rowFilter);
s1 = new Scan();
s1.setFilter(filterList);
scanner = testRegion.getScanner(s1);
results = new ArrayList<>();
for (i=0; i<=4; i+=2) {
scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
for (i=5; i<=9; i++) {
scanner.next(results);
assertTrue(CellUtil.matchingRows(results.get(0), Bytes.toBytes("row" + i)));
assertEquals(Bytes.toInt(CellUtil.cloneValue(results.get(0))), i%2);
results.clear();
}
assertFalse(scanner.next(results));
WAL wal = ((HRegion)testRegion).getWAL();
((HRegion)testRegion).close();
wal.close();
}
}
