Java Code Examples for org.apache.hadoop.hbase.filter.Filter

The following are top voted examples for showing how to use org.apache.hadoop.hbase.filter.Filter. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: ditb   File: TestTableInputFormat.java   Source Code and License 7 votes vote down vote up
@Override
public void configure(JobConf job) {
  try {
    Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job));
    TableName tableName = TableName.valueOf("exampleJobConfigurableTable");
    // mandatory
    initializeTable(connection, tableName);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    //optional
    Scan scan = new Scan();
    for (byte[] family : inputColumns) {
      scan.addFamily(family);
    }
    Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
    scan.setFilter(exampleFilter);
    setScan(scan);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to initialize.", exception);
  }
}
 
Example 2
Project: SparkDemo   File: MyClass.java   Source Code and License 6 votes vote down vote up
public static void QueryByCondition2(String tableName) {

        try {
            HTablePool pool = new HTablePool(configuration, 1000);
            HTable table = (HTable) pool.getTable(tableName);
            Filter filter = new SingleColumnValueFilter(Bytes
                    .toBytes("column1"), null, CompareOp.EQUAL, Bytes
                    .toBytes("aaa")); // 当列column1的值为aaa时进行查询
            Scan s = new Scan();
            s.setFilter(filter);
            ResultScanner rs = table.getScanner(s);
            for (Result r : rs) {
                System.out.println("获得到rowkey:" + new String(r.getRow()));
                for (KeyValue keyValue : r.raw()) {
                    System.out.println("列:" + new String(keyValue.getFamily())
                            + "====值:" + new String(keyValue.getValue()));
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }

    }
 
Example 3
Project: QDrill   File: HBaseFilterBuilder.java   Source Code and License 6 votes vote down vote up
private HBaseScanSpec mergeScanSpecs(String functionName, HBaseScanSpec leftScanSpec, HBaseScanSpec rightScanSpec) {
  Filter newFilter = null;
  byte[] startRow = HConstants.EMPTY_START_ROW;
  byte[] stopRow = HConstants.EMPTY_END_ROW;

  switch (functionName) {
  case "booleanAnd":
    newFilter = HBaseUtils.andFilterAtIndex(leftScanSpec.filter, HBaseUtils.LAST_FILTER, rightScanSpec.filter);
    startRow = HBaseUtils.maxOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.minOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
    break;
  case "booleanOr":
    newFilter = HBaseUtils.orFilterAtIndex(leftScanSpec.filter, HBaseUtils.LAST_FILTER, rightScanSpec.filter);
    startRow = HBaseUtils.minOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
  }
  return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter);
}
 
Example 4
Project: QDrill   File: HBaseUtils.java   Source Code and License 6 votes vote down vote up
public static Filter andFilterAtIndex(Filter currentFilter, int index, Filter newFilter) {
  if (currentFilter == null) {
    return newFilter;
  } else if (newFilter == null) {
    return currentFilter;
  }

  List<Filter> allFilters = Lists.newArrayList();
  if (currentFilter instanceof FilterList && ((FilterList)currentFilter).getOperator() == FilterList.Operator.MUST_PASS_ALL) {
    allFilters.addAll(((FilterList)currentFilter).getFilters());
  } else {
    allFilters.add(currentFilter);
  }
  allFilters.add((index == LAST_FILTER ? allFilters.size() : index), newFilter);
  return new FilterList(FilterList.Operator.MUST_PASS_ALL, allFilters);
}
 
Example 5
Project: QDrill   File: HBaseUtils.java   Source Code and License 6 votes vote down vote up
public static Filter orFilterAtIndex(Filter currentFilter, int index, Filter newFilter) {
  if (currentFilter == null) {
    return newFilter;
  } else if (newFilter == null) {
    return currentFilter;
  }

  List<Filter> allFilters = Lists.newArrayList();
  if (currentFilter instanceof FilterList && ((FilterList)currentFilter).getOperator() == FilterList.Operator.MUST_PASS_ONE) {
    allFilters.addAll(((FilterList)currentFilter).getFilters());
  } else {
    allFilters.add(currentFilter);
  }
  allFilters.add((index == LAST_FILTER ? allFilters.size() : index), newFilter);
  return new FilterList(FilterList.Operator.MUST_PASS_ONE, allFilters);
}
 
Example 6
Project: dremio-oss   File: HBaseFilterBuilder.java   Source Code and License 6 votes vote down vote up
public HBaseScanSpec parseTree() {
  HBaseScanSpec parsedSpec = le.accept(this, null);
  if (parsedSpec != null) {
    parsedSpec = mergeScanSpecs("booleanAnd", this.groupScan.getHBaseScanSpec(), parsedSpec);
    /*
     * If RowFilter is THE filter attached to the scan specification,
     * remove it since its effect is also achieved through startRow and stopRow.
     */
    Filter parsedFilter = HBaseUtils.deserializeFilter(parsedSpec.filter);
    if (parsedFilter instanceof RowFilter &&
        ((RowFilter)parsedFilter).getComparator() instanceof BinaryComparator) {
      parsedSpec.filter = null;
    }
  }
  return parsedSpec;
}
 
Example 7
Project: dremio-oss   File: HBaseFilterBuilder.java   Source Code and License 6 votes vote down vote up
private HBaseScanSpec mergeScanSpecs(String functionName, HBaseScanSpec leftScanSpec, HBaseScanSpec rightScanSpec) {
  Filter newFilter = null;
  byte[] startRow = HConstants.EMPTY_START_ROW;
  byte[] stopRow = HConstants.EMPTY_END_ROW;

  switch (functionName) {
  case "booleanAnd":
    newFilter = HBaseUtils.andFilterAtIndex(
        HBaseUtils.deserializeFilter(leftScanSpec.filter),
        HBaseUtils.LAST_FILTER,
        HBaseUtils.deserializeFilter(rightScanSpec.filter));
    startRow = HBaseUtils.maxOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.minOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
    break;
  case "booleanOr":
    newFilter = HBaseUtils.orFilterAtIndex(
        HBaseUtils.deserializeFilter(leftScanSpec.filter),
        HBaseUtils.LAST_FILTER,
        HBaseUtils.deserializeFilter(rightScanSpec.filter));
    startRow = HBaseUtils.minOfStartRows(leftScanSpec.startRow, rightScanSpec.startRow);
    stopRow = HBaseUtils.maxOfStopRows(leftScanSpec.stopRow, rightScanSpec.stopRow);
  }
  return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, newFilter);
}
 
Example 8
Project: aliyun-tablestore-hbase-client   File: TestFilterList.java   Source Code and License 6 votes vote down vote up
@Test
public void testTwoFilterWithMustAllPassFailed() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_1"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_1_var"));
        Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_2"),
                CompareFilter.CompareOp.NOT_EQUAL, Bytes.toBytes("col_2_var"));
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);

        get.setFilter(filterList);
        Result result = table.get(get);
        assertTrue(result.getRow() == null);
    }
}
 
Example 9
Project: aliyun-tablestore-hbase-client   File: TestFilterList.java   Source Code and License 6 votes vote down vote up
@Test
public void testTwoFilterWithMustOnePassFailed() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_1"),
                CompareFilter.CompareOp.NOT_EQUAL, Bytes.toBytes("col_1_var"));
        Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_2"),
                CompareFilter.CompareOp.NOT_EQUAL, Bytes.toBytes("col_2_var"));
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);

        get.setFilter(filterList);
        Result result = table.get(get);
        assertTrue(result.getRow() == null);
    }
}
 
Example 10
Project: ditb   File: CellCounter.java   Source Code and License 6 votes vote down vote up
private static Scan getConfiguredScanForJob(Configuration conf, String[] args) throws IOException {
  Scan s = new Scan();
  // Set Scan Versions
  s.setMaxVersions(Integer.MAX_VALUE);
  s.setCacheBlocks(false);
  // Set Scan Column Family
  if (conf.get(TableInputFormat.SCAN_COLUMN_FAMILY) != null) {
    s.addFamily(Bytes.toBytes(conf.get(TableInputFormat.SCAN_COLUMN_FAMILY)));
  }
  // Set RowFilter or Prefix Filter if applicable.
  Filter rowFilter = getRowFilter(args);
  if (rowFilter!= null) {
    LOG.info("Setting Row Filter for counter.");
    s.setFilter(rowFilter);
  }
  // Set TimeRange if defined
  long timeRange[] = getTimeRange(args);
  if (timeRange != null) {
    LOG.info("Setting TimeRange for counter.");
    s.setTimeRange(timeRange[0], timeRange[1]);
  }
  return s;
}
 
Example 11
Project: ditb   File: ScanPreprocess.java   Source Code and License 6 votes vote down vote up
public static ConditionTree preprocess(HRegion region, Filter filter, float maxScale) {
  if (filter == null) return null;
  ConditionTree tree = null;
  if (isIndexFilter(region, filter)) {
    System.out.println("preprocess A");
    tree = new ConditionTreeNoneLeafNode(region, (SingleColumnValueFilter) filter, maxScale);
  } else if (filter instanceof FilterList) {
    System.out.println("preprocess B");
    tree = new ConditionTreeNoneLeafNode(region, (FilterList) filter, maxScale);
  }
  if (tree.isPrune()) {
    System.out.println("return null for prune");
    return null;
  } else {
    return tree;
  }
}
 
Example 12
Project: ditb   File: PerformanceEvaluation.java   Source Code and License 6 votes vote down vote up
protected Scan constructScan(byte[] valuePrefix) throws IOException {
  FilterList list = new FilterList();
  Filter filter = new SingleColumnValueFilter(
      FAMILY_NAME, COLUMN_ZERO, CompareFilter.CompareOp.EQUAL,
      new BinaryComparator(valuePrefix)
  );
  list.addFilter(filter);
  if(opts.filterAll) {
    list.addFilter(new FilterAllFilter());
  }
  Scan scan = new Scan();
  scan.setCaching(opts.caching);
  if (opts.addColumns) {
    scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
  } else {
    scan.addFamily(FAMILY_NAME);
  }
  scan.setFilter(list);
  return scan;
}
 
Example 13
Project: ditb   File: TestTableInputFormat.java   Source Code and License 6 votes vote down vote up
@Override
public void configure(JobConf job) {
  try {
    HTable exampleTable = new HTable(HBaseConfiguration.create(job),
      Bytes.toBytes("exampleDeprecatedTable"));
    // mandatory
    setHTable(exampleTable);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    // mandatory
    setInputColumns(inputColumns);
    Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
    // optional
    setRowFilter(exampleFilter);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to configure for job.", exception);
  }
}
 
Example 14
Project: ditb   File: TestTableInputFormat.java   Source Code and License 6 votes vote down vote up
@Override
public void configure(JobConf job) {
  try {
    HTable exampleTable = new HTable(HBaseConfiguration.create(job),
      Bytes.toBytes("exampleDeprecatedTable"));
    // mandatory
    setHTable(exampleTable);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    // optional
    Scan scan = new Scan();
    for (byte[] family : inputColumns) {
      scan.addFamily(family);
    }
    Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
    scan.setFilter(exampleFilter);
    setScan(scan);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to configure for job.", exception);
  }
}
 
Example 15
Project: ditb   File: TestTableInputFormat.java   Source Code and License 6 votes vote down vote up
@Override
protected void initialize(JobContext job) throws IOException {
  Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(
      job.getConfiguration()));
  TableName tableName = TableName.valueOf("exampleTable");
  // mandatory
  initializeTable(connection, tableName);
  byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
    Bytes.toBytes("columnB") };
  //optional
  Scan scan = new Scan();
  for (byte[] family : inputColumns) {
    scan.addFamily(family);
  }
  Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
  scan.setFilter(exampleFilter);
  setScan(scan);
}
 
Example 16
Project: ditb   File: ProtobufUtil.java   Source Code and License 6 votes vote down vote up
/**
 * Convert a protocol buffer Filter to a client Filter
 *
 * @param proto the protocol buffer Filter to convert
 * @return the converted Filter
 */
@SuppressWarnings("unchecked")
public static Filter toFilter(FilterProtos.Filter proto) throws IOException {
  String type = proto.getName();
  final byte [] value = proto.getSerializedFilter().toByteArray();
  String funcName = "parseFrom";
  try {
    Class<? extends Filter> c =
      (Class<? extends Filter>)Class.forName(type, true, CLASS_LOADER);
    Method parseFrom = c.getMethod(funcName, byte[].class);
    if (parseFrom == null) {
      throw new IOException("Unable to locate function: " + funcName + " in type: " + type);
    }
    return (Filter)parseFrom.invoke(c, value);
  } catch (Exception e) {
    // Either we couldn't instantiate the method object, or "parseFrom" failed.
    // In either case, let's not retry.
    throw new DoNotRetryIOException(e);
  }
}
 
Example 17
Project: ditb   File: ScannerModel.java   Source Code and License 5 votes vote down vote up
/**
 * @param filter the filter
 * @return the JSON representation of the filter
 * @throws Exception 
 */
public static String stringifyFilter(final Filter filter) throws Exception {
  JSONJAXBContext context =
    new JSONJAXBContext(JSONConfiguration.natural().build(),
      FilterModel.class);
  JSONMarshaller marshaller = context.createJSONMarshaller();
  StringWriter writer = new StringWriter();
  marshaller.marshallToJSON(new FilterModel(filter), writer);
  return writer.toString();
}
 
Example 18
Project: big_data   File: ActiveUserRunner.java   Source Code and License 5 votes vote down vote up
/**
 * 获取这个列名过滤的column
 * 
 * @param columns
 * @return
 */
private Filter getColumnFilter(String[] columns) {
	int length = columns.length;
	byte[][] filter = new byte[length][];
	for (int i = 0; i < length; i++) {
		filter[i] = Bytes.toBytes(columns[i]);
	}
	return new MultipleColumnPrefixFilter(filter);
}
 
Example 19
Project: QDrill   File: HBaseFilterBuilder.java   Source Code and License 5 votes vote down vote up
private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call,
    CompareFunctionsProcessor processor) {
    byte[] startRow = processor.getRowKeyPrefixStartRow();
    byte[] stopRow  = processor.getRowKeyPrefixStopRow();
    Filter filter   = processor.getRowKeyPrefixFilter();

    if (startRow != HConstants.EMPTY_START_ROW ||
      stopRow != HConstants.EMPTY_END_ROW ||
      filter != null) {
      return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter);
    }

    // else
    return null;
}
 
Example 20
Project: QDrill   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
static Filter parseFilterString(String filterString) {
  if (filterString == null) {
    return null;
  }
  try {
    return FILTER_PARSEER.parseFilterString(filterString);
  } catch (CharacterCodingException e) {
    throw new DrillRuntimeException("Error parsing filter string: " + filterString, e);
  }
}
 
Example 21
Project: QDrill   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
public static byte[] serializeFilter(Filter filter) {
  if (filter == null) {
    return null;
  }
  try {
    FilterProtos.Filter pbFilter = ProtobufUtil.toFilter(filter);
    return pbFilter.toByteArray();
  } catch (IOException e) {
    throw new DrillRuntimeException("Error serializing filter: " + filter, e);
  }
}
 
Example 22
Project: QDrill   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
public static Filter deserializeFilter(byte[] filterBytes) {
  if (filterBytes == null) {
    return null;
  }
  try {
    FilterProtos.Filter pbFilter = FilterProtos.Filter.parseFrom(filterBytes);
    return ProtobufUtil.toFilter(pbFilter);
  } catch (Exception e) {
    throw new DrillRuntimeException("Error deserializing filter: " + filterBytes, e);
  }
}
 
Example 23
Project: ignite-hbase   File: HBaseCacheStore.java   Source Code and License 5 votes vote down vote up
private Scan createScan(Object[] args) {
  Scan scan = new Scan();
  scan.addColumn(family(), QUALIFIER);
  Optional<Filter> filter = FilterParser.createFilter(args);
  filter.ifPresent(scan::setFilter);
  return scan;
}
 
Example 24
Project: hadoop-ecosystem-examples   File: HBaseService.java   Source Code and License 5 votes vote down vote up
/**
 * 多条件查询
 * @param tableName
 * @param familyNames
 * @param qualifiers
 * @param values
    * @throws IOException
    */
public static void queryByConditions(String tableName, String[] familyNames, String[] qualifiers,String[] values) throws IOException {
	Connection conn = ConnectionFactory.createConnection(conf);
	Table table = conn.getTable(TableName.valueOf(tableName));
	try {
		List<Filter> filters = new ArrayList<Filter>();
		if (familyNames != null && familyNames.length > 0) {
			int i = 0;
			for (String familyName : familyNames) {
				Filter filter = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes(qualifiers[i]), CompareFilter.CompareOp.EQUAL, Bytes.toBytes(values[i]));
				filters.add(filter);
				i++;
			}
		}
		FilterList filterList = new FilterList(filters);
		Scan scan = new Scan();
		scan.setFilter(filterList);
		ResultScanner rs = table.getScanner(scan);
		for (Result r : rs) {
			System.out.println("获得到rowkey:" + new String(r.getRow()));
			for (Cell keyValue : r.rawCells()) {
				System.out.println("列:" + new String(CellUtil.cloneFamily(keyValue))+":"+new String(CellUtil.cloneQualifier(keyValue)) + "====值:" + new String(CellUtil.cloneValue(keyValue)));
			}
		}
		rs.close();
	} catch (Exception e) {
		table.close();
		conn.close();
	}

}
 
Example 25
Project: dremio-oss   File: HBaseFilterBuilder.java   Source Code and License 5 votes vote down vote up
private HBaseScanSpec createRowKeyPrefixScanSpec(FunctionCall call, CompareFunctionsProcessor processor) {
  byte[] startRow = processor.getRowKeyPrefixStartRow();
  byte[] stopRow  = processor.getRowKeyPrefixStopRow();
  Filter filter   = processor.getRowKeyPrefixFilter();

  if (startRow != HConstants.EMPTY_START_ROW ||
      stopRow != HConstants.EMPTY_END_ROW ||
      filter != null) {
    return new HBaseScanSpec(groupScan.getTableName(), startRow, stopRow, filter);
  }

  // else
  return null;
}
 
Example 26
Project: dremio-oss   File: HBaseScanSpec.java   Source Code and License 5 votes vote down vote up
public HBaseScanSpec(String tableName, byte[] startRow, byte[] stopRow, Filter filter) {
  this.tableName = tableName;
  this.startRow = startRow;
  this.stopRow = stopRow;
  if (filter != null) {
    this.filter = HBaseUtils.serializeFilter(filter);
  } else {
    this.filter = null;
  }
}
 
Example 27
Project: dremio-oss   File: HBaseScanSpec.java   Source Code and License 5 votes vote down vote up
@JsonIgnore
public Filter getFilter() {
  if (filterParsed == null) {
    synchronized(this) {
      if (filterParsed == null) {
        filterParsed = HBaseUtils.deserializeFilter(this.filter);
      }
    }
  }
  return filterParsed;
}
 
Example 28
Project: dremio-oss   File: HBaseScanSpec.java   Source Code and License 5 votes vote down vote up
@Override
public String toString() {
  Filter filterToString = getFilter();
  return "HBaseScanSpec [tableName=" + tableName
      + ", startRow=" + (startRow == null ? null : Bytes.toStringBinary(startRow))
      + ", stopRow=" + (stopRow == null ? null : Bytes.toStringBinary(stopRow))
      + ", filter=" + (filter == null ? null : filterToString.toString())
      + "]";
}
 
Example 29
Project: dremio-oss   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
static Filter parseFilterString(String filterString) {
  if (filterString == null) {
    return null;
  }
  try {
    return FILTER_PARSEER.parseFilterString(filterString);
  } catch (CharacterCodingException e) {
    throw new RuntimeException("Error parsing filter string: " + filterString, e);
  }
}
 
Example 30
Project: dremio-oss   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
public static byte[] serializeFilter(Filter filter) {
  if (filter == null) {
    return null;
  }
  try {
    FilterProtos.Filter pbFilter = ProtobufUtil.toFilter(filter);
    return pbFilter.toByteArray();
  } catch (IOException e) {
    throw new RuntimeException("Error serializing filter: " + filter, e);
  }
}
 
Example 31
Project: dremio-oss   File: HBaseUtils.java   Source Code and License 5 votes vote down vote up
public static Filter deserializeFilter(byte[] filterBytes) {
  if (filterBytes == null) {
    return null;
  }
  try {
    FilterProtos.Filter pbFilter = FilterProtos.Filter.parseFrom(filterBytes);
    return ProtobufUtil.toFilter(pbFilter);
  } catch (Exception e) {
    throw new RuntimeException("Error deserializing filter: " + filterBytes, e);
  }
}
 
Example 32
Project: aliyun-tablestore-hbase-client   File: TestColumnPaginationFilter.java   Source Code and License 5 votes vote down vote up
@Test
public void testResultLessThanLimit() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter = new ColumnPaginationFilter(5, 3);
        get.setFilter(filter);
        Result result = table.get(get);
        List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4"));
        assertEquals(1, cells.size());
        assertEquals("col_4_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength()));
    }
}
 
Example 33
Project: aliyun-tablestore-hbase-client   File: TestColumnPaginationFilter.java   Source Code and License 5 votes vote down vote up
@Test
public void testStartRow() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_2"));
        get.setFilter(filter);
        Result result = table.get(get);
        List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2"));
        assertEquals(1, cells.size());
        assertEquals("col_2_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength()));

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4"));
        assertEquals(0, cells.size());
    }
}
 
Example 34
Project: aliyun-tablestore-hbase-client   File: TestColumnPaginationFilter.java   Source Code and License 5 votes vote down vote up
@Test
public void testStartRowByScan() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var"));
        table.put(put);
    }

    {
        Scan scan = new Scan();
        Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_2"));
        scan.setFilter(filter);
        scan.setStartRow(Bytes.toBytes(rowPrefix));
        scan.setStopRow(Bytes.toBytes(rowPrefix + 5));
        ResultScanner scanResult = table.getScanner(scan);

        Result result = scanResult.next();

        List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2"));
        assertEquals(1, cells.size());
        assertEquals("col_2_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength()));

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4"));
        assertEquals(0, cells.size());
    }
}
 
Example 35
Project: aliyun-tablestore-hbase-client   File: TestColumnPaginationFilter.java   Source Code and License 5 votes vote down vote up
@Test
public void testStartRowIsNotExist() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_3"), Bytes.toBytes("col_3_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_4"), Bytes.toBytes("col_4_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter = new ColumnPaginationFilter(1, Bytes.toBytes("col_3_not_exist"));
        get.setFilter(filter);
        Result result = table.get(get);
        List<Cell> cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_1"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_2"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_3"));
        assertEquals(0, cells.size());

        cells = result.getColumnCells(Bytes.toBytes(familyName), Bytes.toBytes("col_4"));
        assertEquals(1, cells.size());
        assertEquals("col_4_var", Bytes.toString(cells.get(0).getValueArray(), cells.get(0).getValueOffset(), cells.get(0).getValueLength()));
    }
}
 
Example 36
Project: aliyun-tablestore-hbase-client   File: TestFilterList.java   Source Code and License 5 votes vote down vote up
@Test
public void testTwoFilterWithDefaultOperator() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_1"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_1_var"));
        Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_2"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_2_var"));
        FilterList filterList = new FilterList();
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);

        get.setFilter(filterList);
        Result result = table.get(get);
        String value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_1")));
        assertEquals("col_1_var", value);
        value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_2")));
        assertEquals("col_2_var", value);
    }
}
 
Example 37
Project: aliyun-tablestore-hbase-client   File: TestFilterList.java   Source Code and License 5 votes vote down vote up
@Test
public void testTwoFilterWithMustAllPassSucceeded() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_1"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_1_var"));
        Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_2"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_2_var"));
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);

        get.setFilter(filterList);
        Result result = table.get(get);
        String value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_1")));
        assertEquals("col_1_var", value);
        value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_2")));
        assertEquals("col_2_var", value);
    }
}
 
Example 38
Project: aliyun-tablestore-hbase-client   File: TestFilterList.java   Source Code and License 5 votes vote down vote up
@Test
public void testTwoFilterWithMustOnePassSucceeded() throws IOException {
    clean();
    {
        Put put = new Put(Bytes.toBytes(rowPrefix));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_1"), Bytes.toBytes("col_1_var"));
        put.addColumn(Bytes.toBytes(familyName), Bytes.toBytes("col_2"), Bytes.toBytes("col_2_var"));
        table.put(put);
    }

    {
        Get get = new Get(Bytes.toBytes(rowPrefix));
        Filter filter1 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_1"),
                CompareFilter.CompareOp.EQUAL, Bytes.toBytes("col_1_var"));
        Filter filter2 = new SingleColumnValueFilter(Bytes.toBytes(familyName), Bytes.toBytes("col_2"),
                CompareFilter.CompareOp.NOT_EQUAL, Bytes.toBytes("col_2_var"));
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ONE);
        filterList.addFilter(filter1);
        filterList.addFilter(filter2);

        get.setFilter(filterList);
        Result result = table.get(get);
        String value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_1")));
        assertEquals("col_1_var", value);
        value = Bytes.toString(result.getValue(Bytes.toBytes(familyName), Bytes.toBytes("col_2")));
        assertEquals("col_2_var", value);
    }
}
 
Example 39
Project: ditb   File: ThriftServerRunner.java   Source Code and License 5 votes vote down vote up
@Override
public int scannerOpenWithPrefix(ByteBuffer tableName,
                                 ByteBuffer startAndPrefix,
                                 List<ByteBuffer> columns,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan(getBytes(startAndPrefix));
    addAttributes(scan, attributes);
    Filter f = new WhileMatchFilter(
        new PrefixFilter(getBytes(startAndPrefix)));
    scan.setFilter(f);
    if (columns != null && columns.size() != 0) {
      for(ByteBuffer column : columns) {
        byte [][] famQf = KeyValue.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw new IOError(Throwables.getStackTraceAsString(e));
  } finally{
    closeTable(table);
  }
}
 
Example 40
Project: ditb   File: VisibilityController.java   Source Code and License 5 votes vote down vote up
@Override
public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
    RegionScanner s) throws IOException {
  if (!initialized) {
    throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized!");
  }
  // Nothing to do if authorization is not enabled
  if (!authorizationEnabled) {
    return s;
  }
  Region region = e.getEnvironment().getRegion();
  Authorizations authorizations = null;
  try {
    authorizations = scan.getAuthorizations();
  } catch (DeserializationException de) {
    throw new IOException(de);
  }
  if (authorizations == null) {
    // No Authorizations present for this scan/Get!
    // In case of system tables other than "labels" just scan with out visibility check and
    // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
    TableName table = region.getRegionInfo().getTable();
    if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
      return s;
    }
  }

  Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(region,
      authorizations);
  if (visibilityLabelFilter != null) {
    Filter filter = scan.getFilter();
    if (filter != null) {
      scan.setFilter(new FilterList(filter, visibilityLabelFilter));
    } else {
      scan.setFilter(visibilityLabelFilter);
    }
  }
  return s;
}
 
Example 41
Project: ditb   File: VisibilityController.java   Source Code and License 5 votes vote down vote up
@Override
public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e, Get get,
    List<Cell> results) throws IOException {
  if (!initialized) {
    throw new VisibilityControllerNotReadyException("VisibilityController not yet initialized");
  }
  // Nothing useful to do if authorization is not enabled
  if (!authorizationEnabled) {
    return;
  }
  Region region = e.getEnvironment().getRegion();
  Authorizations authorizations = null;
  try {
    authorizations = get.getAuthorizations();
  } catch (DeserializationException de) {
    throw new IOException(de);
  }
  if (authorizations == null) {
    // No Authorizations present for this scan/Get!
    // In case of system tables other than "labels" just scan with out visibility check and
    // filtering. Checking visibility labels for META and NAMESPACE table is not needed.
    TableName table = region.getRegionInfo().getTable();
    if (table.isSystemTable() && !table.equals(LABELS_TABLE_NAME)) {
      return;
    }
  }
  Filter visibilityLabelFilter = VisibilityUtils.createVisibilityLabelFilter(e.getEnvironment()
      .getRegion(), authorizations);
  if (visibilityLabelFilter != null) {
    Filter filter = get.getFilter();
    if (filter != null) {
      get.setFilter(new FilterList(filter, visibilityLabelFilter));
    } else {
      get.setFilter(visibilityLabelFilter);
    }
  }
}
 
Example 42
Project: ditb   File: VisibilityUtils.java   Source Code and License 5 votes vote down vote up
public static Filter createVisibilityLabelFilter(Region region, Authorizations authorizations)
    throws IOException {
  Map<ByteRange, Integer> cfVsMaxVersions = new HashMap<ByteRange, Integer>();
  for (HColumnDescriptor hcd : region.getTableDesc().getFamilies()) {
    cfVsMaxVersions.put(new SimpleMutableByteRange(hcd.getName()), hcd.getMaxVersions());
  }
  VisibilityLabelService vls = VisibilityLabelServiceManager.getInstance()
      .getVisibilityLabelService();
  Filter visibilityLabelFilter = new VisibilityLabelFilter(
      vls.getVisibilityExpEvaluator(authorizations), cfVsMaxVersions);
  return visibilityLabelFilter;
}
 
Example 43
Project: ditb   File: DefaultVisibilityLabelServiceImpl.java   Source Code and License 5 votes vote down vote up
@Override
public List<String> getUserAuths(byte[] user, boolean systemCall)
    throws IOException {
  assert (labelsRegion != null || systemCall);
  if (systemCall || labelsRegion == null) {
    return this.labelsCache.getUserAuths(Bytes.toString(user));
  }
  Scan s = new Scan();
  if (user != null && user.length > 0) {
    s.addColumn(LABELS_TABLE_FAMILY, user);
  }
  Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
    new Authorizations(SYSTEM_LABEL));
  s.setFilter(filter);
  ArrayList<String> auths = new ArrayList<String>();
  RegionScanner scanner = this.labelsRegion.getScanner(s);
  try {
    List<Cell> results = new ArrayList<Cell>(1);
    while (true) {
      scanner.next(results);
      if (results.isEmpty()) break;
      Cell cell = results.get(0);
      int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
      String label = this.labelsCache.getLabel(ordinal);
      if (label != null) {
        auths.add(label);
      }
      results.clear();
    }
  } finally {
    scanner.close();
  }
  return auths;
}
 
Example 44
Project: ditb   File: DefaultVisibilityLabelServiceImpl.java   Source Code and License 5 votes vote down vote up
@Override
public List<String> getGroupAuths(String[] groups, boolean systemCall)
    throws IOException {
  assert (labelsRegion != null || systemCall);
  if (systemCall || labelsRegion == null) {
    return this.labelsCache.getGroupAuths(groups);
  }
  Scan s = new Scan();
  if (groups != null && groups.length > 0) {
    for (String group : groups) {
      s.addColumn(LABELS_TABLE_FAMILY, Bytes.toBytes(AuthUtil.toGroupEntry(group)));
    }
  }
  Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
      new Authorizations(SYSTEM_LABEL));
  s.setFilter(filter);
  Set<String> auths = new HashSet<String>();
  RegionScanner scanner = this.labelsRegion.getScanner(s);
  try {
    List<Cell> results = new ArrayList<Cell>(1);
    while (true) {
      scanner.next(results);
      if (results.isEmpty()) break;
      Cell cell = results.get(0);
      int ordinal = Bytes.toInt(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
      String label = this.labelsCache.getLabel(ordinal);
      if (label != null) {
        auths.add(label);
      }
      results.clear();
    }
  } finally {
    scanner.close();
  }
  return new ArrayList<String>(auths);
}
 
Example 45
Project: ditb   File: RowResultGenerator.java   Source Code and License 5 votes vote down vote up
public RowResultGenerator(final String tableName, final RowSpec rowspec,
    final Filter filter, final boolean cacheBlocks)
    throws IllegalArgumentException, IOException {
  Table table = RESTServlet.getInstance().getTable(tableName);
  try {
    Get get = new Get(rowspec.getRow());
    if (rowspec.hasColumns()) {
      for (byte[] col: rowspec.getColumns()) {
        byte[][] split = KeyValue.parseColumn(col);
        if (split.length == 1) {
          get.addFamily(split[0]);
        } else if (split.length == 2) {
          get.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid column specifier.");
        }
      }
    }
    get.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    get.setMaxVersions(rowspec.getMaxVersions());
    if (filter != null) {
      get.setFilter(filter);
    }
    get.setCacheBlocks(cacheBlocks);
    Result result = table.get(get);
    if (result != null && !result.isEmpty()) {
      valuesI = result.listCells().iterator();
    }
  } catch (DoNotRetryIOException | NeedUnmanagedConnectionException e) {
    // Warn here because Stargate will return 404 in the case if multiple
    // column families were specified but one did not exist -- currently
    // HBase will fail the whole Get.
    // Specifying multiple columns in a URI should be uncommon usage but
    // help to avoid confusion by leaving a record of what happened here in
    // the log.
    LOG.warn(StringUtils.stringifyException(e));
  } finally {
    table.close();
  }
}
 
Example 46
Project: ditb   File: ProtobufUtil.java   Source Code and License 5 votes vote down vote up
/**
 * Convert a client Filter to a protocol buffer Filter
 *
 * @param filter the Filter to convert
 * @return the converted protocol buffer Filter
 */
public static FilterProtos.Filter toFilter(Filter filter) throws IOException {
  FilterProtos.Filter.Builder builder = FilterProtos.Filter.newBuilder();
  builder.setName(filter.getClass().getName());
  builder.setSerializedFilter(ByteStringer.wrap(filter.toByteArray()));
  return builder.build();
}
 
Example 47
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testFilters() throws Exception {
  byte [] TABLE = Bytes.toBytes("testFilters");
  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
  byte [][] ROWS = makeN(ROW, 10);
  byte [][] QUALIFIERS = {
      Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
      Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
      Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
      Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
      Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
  };
  for(int i=0;i<10;i++) {
    Put put = new Put(ROWS[i]);
    put.setDurability(Durability.SKIP_WAL);
    put.add(FAMILY, QUALIFIERS[i], VALUE);
    ht.put(put);
  }
  Scan scan = new Scan();
  scan.addFamily(FAMILY);
  Filter filter = new QualifierFilter(CompareOp.EQUAL,
    new RegexStringComparator("col[1-5]"));
  scan.setFilter(filter);
  ResultScanner scanner = ht.getScanner(scan);
  int expectedIndex = 1;
  for(Result result : ht.getScanner(scan)) {
    assertEquals(result.size(), 1);
    assertTrue(Bytes.equals(CellUtil.cloneRow(result.rawCells()[0]), ROWS[expectedIndex]));
    assertTrue(Bytes.equals(CellUtil.cloneQualifier(result.rawCells()[0]),
        QUALIFIERS[expectedIndex]));
    expectedIndex++;
  }
  assertEquals(expectedIndex, 6);
  scanner.close();
}
 
Example 48
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
  public void testFilterWithLongCompartor() throws Exception {
    byte [] TABLE = Bytes.toBytes("testFilterWithLongCompartor");
    Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
    byte [][] ROWS = makeN(ROW, 10);
    byte [][] values = new byte[10][];
    for (int i = 0; i < 10; i ++) {
        values[i] = Bytes.toBytes(100L * i);
    }
    for(int i = 0; i < 10; i ++) {
      Put put = new Put(ROWS[i]);
      put.setDurability(Durability.SKIP_WAL);
      put.add(FAMILY, QUALIFIER, values[i]);
      ht.put(put);
    }
    Scan scan = new Scan();
    scan.addFamily(FAMILY);
    Filter filter = new SingleColumnValueFilter(FAMILY, QUALIFIER, CompareOp.GREATER,
      new LongComparator(500));
    scan.setFilter(filter);
    ResultScanner scanner = ht.getScanner(scan);
    int expectedIndex = 0;
    for(Result result : ht.getScanner(scan)) {
      assertEquals(result.size(), 1);
      assertTrue(Bytes.toLong(result.getValue(FAMILY, QUALIFIER)) > 500);
      expectedIndex++;
    }
    assertEquals(expectedIndex, 4);
    scanner.close();
}
 
Example 49
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testKeyOnlyFilter() throws Exception {
  byte [] TABLE = Bytes.toBytes("testKeyOnlyFilter");
  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
  byte [][] ROWS = makeN(ROW, 10);
  byte [][] QUALIFIERS = {
      Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
      Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
      Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
      Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
      Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>")
  };
  for(int i=0;i<10;i++) {
    Put put = new Put(ROWS[i]);
    put.setDurability(Durability.SKIP_WAL);
    put.add(FAMILY, QUALIFIERS[i], VALUE);
    ht.put(put);
  }
  Scan scan = new Scan();
  scan.addFamily(FAMILY);
  Filter filter = new KeyOnlyFilter(true);
  scan.setFilter(filter);
  ResultScanner scanner = ht.getScanner(scan);
  int count = 0;
  for(Result result : ht.getScanner(scan)) {
    assertEquals(result.size(), 1);
    assertEquals(result.rawCells()[0].getValueLength(), Bytes.SIZEOF_INT);
    assertEquals(Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])), VALUE.length);
    count++;
  }
  assertEquals(count, 10);
  scanner.close();
}
 
Example 50
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testFiltersWithReverseScan() throws Exception {
  TableName TABLE = TableName.valueOf("testFiltersWithReverseScan");
  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
  byte[][] ROWS = makeN(ROW, 10);
  byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
      Bytes.toBytes("col1-<d2v1>-<d3v2>"),
      Bytes.toBytes("col2-<d2v1>-<d3v2>"),
      Bytes.toBytes("col3-<d2v1>-<d3v2>"),
      Bytes.toBytes("col4-<d2v1>-<d3v2>"),
      Bytes.toBytes("col5-<d2v1>-<d3v2>"),
      Bytes.toBytes("col6-<d2v1>-<d3v2>"),
      Bytes.toBytes("col7-<d2v1>-<d3v2>"),
      Bytes.toBytes("col8-<d2v1>-<d3v2>"),
      Bytes.toBytes("col9-<d2v1>-<d3v2>") };
  for (int i = 0; i < 10; i++) {
    Put put = new Put(ROWS[i]);
    put.add(FAMILY, QUALIFIERS[i], VALUE);
    ht.put(put);
  }
  Scan scan = new Scan();
  scan.setReversed(true);
  scan.addFamily(FAMILY);
  Filter filter = new QualifierFilter(CompareOp.EQUAL,
      new RegexStringComparator("col[1-5]"));
  scan.setFilter(filter);
  ResultScanner scanner = ht.getScanner(scan);
  int expectedIndex = 5;
  for (Result result : scanner) {
    assertEquals(result.size(), 1);
    assertTrue(Bytes.equals(result.raw()[0].getRow(), ROWS[expectedIndex]));
    assertTrue(Bytes.equals(result.raw()[0].getQualifier(),
        QUALIFIERS[expectedIndex]));
    expectedIndex--;
  }
  assertEquals(expectedIndex, 0);
  scanner.close();
  ht.close();
}
 
Example 51
Project: ditb   File: TestFromClientSide.java   Source Code and License 5 votes vote down vote up
@Test
public void testKeyOnlyFilterWithReverseScan() throws Exception {
  TableName TABLE = TableName.valueOf("testKeyOnlyFilterWithReverseScan");
  Table ht = TEST_UTIL.createTable(TABLE, FAMILY);
  byte[][] ROWS = makeN(ROW, 10);
  byte[][] QUALIFIERS = { Bytes.toBytes("col0-<d2v1>-<d3v2>"),
      Bytes.toBytes("col1-<d2v1>-<d3v2>"),
      Bytes.toBytes("col2-<d2v1>-<d3v2>"),
      Bytes.toBytes("col3-<d2v1>-<d3v2>"),
      Bytes.toBytes("col4-<d2v1>-<d3v2>"),
      Bytes.toBytes("col5-<d2v1>-<d3v2>"),
      Bytes.toBytes("col6-<d2v1>-<d3v2>"),
      Bytes.toBytes("col7-<d2v1>-<d3v2>"),
      Bytes.toBytes("col8-<d2v1>-<d3v2>"),
      Bytes.toBytes("col9-<d2v1>-<d3v2>") };
  for (int i = 0; i < 10; i++) {
    Put put = new Put(ROWS[i]);
    put.add(FAMILY, QUALIFIERS[i], VALUE);
    ht.put(put);
  }
  Scan scan = new Scan();
  scan.setReversed(true);
  scan.addFamily(FAMILY);
  Filter filter = new KeyOnlyFilter(true);
  scan.setFilter(filter);
  ResultScanner scanner = ht.getScanner(scan);
  int count = 0;
  for (Result result : ht.getScanner(scan)) {
    assertEquals(result.size(), 1);
    assertEquals(result.raw()[0].getValueLength(), Bytes.SIZEOF_INT);
    assertEquals(Bytes.toInt(result.raw()[0].getValue()), VALUE.length);
    count++;
  }
  assertEquals(count, 10);
  scanner.close();
  ht.close();
}
 
Example 52
Project: ditb   File: TestTimestampsFilter.java   Source Code and License 5 votes vote down vote up
/**
 * Uses the TimestampFilter on a Get to request a specified list of
 * versions for the row/column specified by rowIdx & colIdx.
 *
 */
private  Cell[] getNVersions(Table ht, byte[] cf, int rowIdx,
                                 int colIdx, List<Long> versions)
  throws IOException {
  byte row[] = Bytes.toBytes("row:" + rowIdx);
  byte column[] = Bytes.toBytes("column:" + colIdx);
  Filter filter = new TimestampsFilter(versions);
  Get get = new Get(row);
  get.addColumn(cf, column);
  get.setFilter(filter);
  get.setMaxVersions();
  Result result = ht.get(get);

  return result.rawCells();
}
 
Example 53
Project: ditb   File: TestTimestampsFilter.java   Source Code and License 5 votes vote down vote up
/**
 * Uses the TimestampFilter on a Scan to request a specified list of
 * versions for the rows from startRowIdx to endRowIdx (both inclusive).
 */
private Result[] scanNVersions(Table ht, byte[] cf, int startRowIdx,
                               int endRowIdx, List<Long> versions)
  throws IOException {
  byte startRow[] = Bytes.toBytes("row:" + startRowIdx);
  byte endRow[] = Bytes.toBytes("row:" + endRowIdx + 1); // exclusive
  Filter filter = new TimestampsFilter(versions);
  Scan scan = new Scan(startRow, endRow);
  scan.setFilter(filter);
  scan.setMaxVersions();
  ResultScanner scanner = ht.getScanner(scan);
  return scanner.next(endRowIdx - startRowIdx + 1);
}
 
Example 54
Project: ditb   File: TestImportExport.java   Source Code and License 5 votes vote down vote up
/**
 * Count the number of keyvalues in the specified table for the given timerange
 * @param start
 * @param end
 * @param table
 * @return
 * @throws IOException
 */
private int getCount(Table table, Filter filter) throws IOException {
  Scan scan = new Scan();
  scan.setFilter(filter);
  ResultScanner results = table.getScanner(scan);
  int count = 0;
  for (Result res : results) {
    count += res.size();
  }
  results.close();
  return count;
}
 
Example 55
Project: ditb   File: ScannerModel.java   Source Code and License 5 votes vote down vote up
/**
 * @param s the JSON representation of the filter
 * @return the filter
 * @throws Exception
 */
public static Filter buildFilter(String s) throws Exception {
  JSONJAXBContext context =
    new JSONJAXBContext(JSONConfiguration.natural().build(),
      FilterModel.class);
  JSONUnmarshaller unmarshaller = context.createJSONUnmarshaller();
  FilterModel model = unmarshaller.unmarshalFromJSON(new StringReader(s),
    FilterModel.class);
  return model.build();
}
 
Example 56
Project: ditb   File: TestAggregateProtocol.java   Source Code and License 5 votes vote down vote up
@Test (timeout=300000)
public void testMaxWithFilter() throws Throwable {
  Long max = 0l;
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
  Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
  scan.setFilter(f);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  max = aClient.max(TEST_TABLE, ci, scan);
  assertEquals(null, max);
}
 
Example 57
Project: ditb   File: TestAggregateProtocol.java   Source Code and License 5 votes vote down vote up
@Test (timeout=300000)
public void testMinWithFilter() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
  Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
  scan.setFilter(f);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Long min = null;
  min = aClient.min(TEST_TABLE, ci, scan);
  assertEquals(null, min);
}
 
Example 58
Project: ditb   File: TestAggregateProtocol.java   Source Code and License 5 votes vote down vote up
@Test (timeout=300000)
public void testSumWithFilter() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setFilter(f);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Long sum = null;
  sum = aClient.sum(TEST_TABLE, ci, scan);
  assertEquals(null, sum);
}
 
Example 59
Project: ditb   File: TestAggregateProtocol.java   Source Code and License 5 votes vote down vote up
@Test (timeout=300000)
public void testAvgWithFilter() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addColumn(TEST_FAMILY,TEST_QUALIFIER);
  Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
  scan.setFilter(f);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Double avg = null;
  avg = aClient.avg(TEST_TABLE, ci, scan);
  assertEquals(Double.NaN, avg, 0);
}
 
Example 60
Project: ditb   File: TestAggregateProtocol.java   Source Code and License 5 votes vote down vote up
@Test (timeout=300000)
public void testStdWithFilter() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Filter f = new PrefixFilter(Bytes.toBytes("foo:bar"));
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setFilter(f);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Double std = null;
  std = aClient.std(TEST_TABLE, ci, scan);
  assertEquals(Double.NaN, std, 0);
}