org.apache.hadoop.hbase.filter.KeyOnlyFilter Java Examples

The following examples show how to use org.apache.hadoop.hbase.filter.KeyOnlyFilter. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RegionLocationCleaner.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}
 
Example #2
Source File: HBaseResourceStore.java    From Kylin with Apache License 2.0 6 votes vote down vote up
private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
    byte[] startRow = Bytes.toBytes(path);
    byte[] endRow = plusZero(startRow);

    Scan scan = new Scan(startRow, endRow);
    if (family == null || column == null) {
        scan.setFilter(new KeyOnlyFilter());
    } else {
        scan.addColumn(family, column);
    }

    HTableInterface table = getConnection().getTable(getAllInOneTableName());
    try {
        ResultScanner scanner = table.getScanner(scan);
        Result result = null;
        for (Result r : scanner) {
            result = r;
        }
        return result == null || result.isEmpty() ? null : result;
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example #3
Source File: IntegrationTestMTTR.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
protected boolean doAction() throws Exception {
  ResultScanner rs = null;
  try {
    Scan s = new Scan();
    s.setBatch(2);
    s.addFamily(FAMILY);
    s.setFilter(new KeyOnlyFilter());
    s.readVersions(1);

    rs = table.getScanner(s);
    Result result = rs.next();
    return result != null && result.size() > 0;
  } finally {
    if (rs != null) {
      rs.close();
    }
  }
}
 
Example #4
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}
 
Example #5
Source File: RegionLocationCleaner.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}
 
Example #6
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}
 
Example #7
Source File: RegionLocationCleaner.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}
 
Example #8
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}
 
Example #9
Source File: RegionLocationCleaner.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
    try (HTableInterface table = connection.getTable(tableName.getBytes())) {
        // Do not use Get not to increase read request count metric.
        // Use Scan.
        Scan scan = new Scan("".getBytes(), "".getBytes());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        //noinspection EmptyTryBlock
        try(ResultScanner ignored = table.getScanner(scan)) {
        }
        return;
    } catch (IOException ignore) {
    }

    clean(tableName);
}
 
Example #10
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static boolean isReallyEmptyRegion(HConnection connection,
    String tableName, HRegionInfo regionInfo) throws IOException {
    boolean emptyRegion = false;
    // verify really empty region by scanning records
    try (HTableInterface table = connection.getTable(tableName)) {
        Scan scan = new Scan(regionInfo.getStartKey(), regionInfo.getEndKey());
        FilterList filterList = new FilterList();
        filterList.addFilter(new KeyOnlyFilter());
        filterList.addFilter(new FirstKeyOnlyFilter());
        scan.setFilter(filterList);
        scan.setCacheBlocks(false);
        scan.setSmall(true);
        scan.setCaching(1);

        try (ResultScanner scanner = table.getScanner(scan)) {
            if (scanner.next() == null) emptyRegion = true;
        }
    }
    return emptyRegion;
}
 
Example #11
Source File: GridTableHBaseBenchmark.java    From kylin-on-parquet-v2 with Apache License 2.0 5 votes vote down vote up
private static void prepareData(Connection conn) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));

    try {
        // check how many rows existing
        int nRows = 0;
        Scan scan = new Scan();
        scan.setFilter(new KeyOnlyFilter());
        ResultScanner scanner = table.getScanner(scan);
        for (Result r : scanner) {
            r.getRow(); // nothing to do
            nRows++;
        }

        if (nRows > 0) {
            logger.info("{} existing rows", nRows);
            if (nRows != N_ROWS)
                throw new IOException("Expect " + N_ROWS + " rows but it is not");
            return;
        }

        // insert rows into empty table
        logger.info("Writing {} rows to {}", N_ROWS, TEST_TABLE);
        long nBytes = 0;
        for (int i = 0; i < N_ROWS; i++) {
            byte[] rowkey = Bytes.toBytes(i);
            Put put = new Put(rowkey);
            byte[] cell = randomBytes();
            put.addColumn(CF, QN, cell);
            table.put(put);
            nBytes += cell.length;
            dot(i, N_ROWS);
        }
        logger.info("Written {} rows, {} bytes", N_ROWS, nBytes);

    } finally {
        IOUtils.closeQuietly(table);
    }

}
 
Example #12
Source File: GridTableHBaseBenchmark.java    From kylin with Apache License 2.0 5 votes vote down vote up
private static void prepareData(Connection conn) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));

    try {
        // check how many rows existing
        int nRows = 0;
        Scan scan = new Scan();
        scan.setFilter(new KeyOnlyFilter());
        ResultScanner scanner = table.getScanner(scan);
        for (Result r : scanner) {
            r.getRow(); // nothing to do
            nRows++;
        }

        if (nRows > 0) {
            logger.info("{} existing rows", nRows);
            if (nRows != N_ROWS)
                throw new IOException("Expect " + N_ROWS + " rows but it is not");
            return;
        }

        // insert rows into empty table
        logger.info("Writing {} rows to {}", N_ROWS, TEST_TABLE);
        long nBytes = 0;
        for (int i = 0; i < N_ROWS; i++) {
            byte[] rowkey = Bytes.toBytes(i);
            Put put = new Put(rowkey);
            byte[] cell = randomBytes();
            put.addColumn(CF, QN, cell);
            table.put(put);
            nBytes += cell.length;
            dot(i, N_ROWS);
        }
        logger.info("Written {} rows, {} bytes", N_ROWS, nBytes);

    } finally {
        IOUtils.closeQuietly(table);
    }

}
 
Example #13
Source File: TestFromClientSide5.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testKeyOnlyFilterWithReverseScan() throws Exception {
  final TableName tableName = name.getTableName();
  try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
    byte[][] ROWS = makeN(ROW, 10);
    byte[][] QUALIFIERS = {Bytes.toBytes("col0-<d2v1>-<d3v2>"),
            Bytes.toBytes("col1-<d2v1>-<d3v2>"),
            Bytes.toBytes("col2-<d2v1>-<d3v2>"),
            Bytes.toBytes("col3-<d2v1>-<d3v2>"),
            Bytes.toBytes("col4-<d2v1>-<d3v2>"),
            Bytes.toBytes("col5-<d2v1>-<d3v2>"),
            Bytes.toBytes("col6-<d2v1>-<d3v2>"),
            Bytes.toBytes("col7-<d2v1>-<d3v2>"),
            Bytes.toBytes("col8-<d2v1>-<d3v2>"),
            Bytes.toBytes("col9-<d2v1>-<d3v2>")};
    for (int i = 0; i < 10; i++) {
      Put put = new Put(ROWS[i]);
      put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
      ht.put(put);
    }
    Scan scan = new Scan();
    scan.setReversed(true);
    scan.addFamily(FAMILY);
    Filter filter = new KeyOnlyFilter(true);
    scan.setFilter(filter);
    try (ResultScanner ignored = ht.getScanner(scan)) {
      int count = 0;
      for (Result result : ht.getScanner(scan)) {
        assertEquals(1, result.size());
        assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
        assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
        count++;
      }
      assertEquals(10, count);
    }
  }
}
 
Example #14
Source File: TestFromClientSide.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test public void testKeyOnlyFilter() throws Exception {
  final TableName tableName = name.getTableName();
  try (Table ht = TEST_UTIL.createTable(tableName, FAMILY)) {
    byte[][] ROWS = makeN(ROW, 10);
    byte[][] QUALIFIERS =
      { Bytes.toBytes("col0-<d2v1>-<d3v2>"), Bytes.toBytes("col1-<d2v1>-<d3v2>"),
        Bytes.toBytes("col2-<d2v1>-<d3v2>"), Bytes.toBytes("col3-<d2v1>-<d3v2>"),
        Bytes.toBytes("col4-<d2v1>-<d3v2>"), Bytes.toBytes("col5-<d2v1>-<d3v2>"),
        Bytes.toBytes("col6-<d2v1>-<d3v2>"), Bytes.toBytes("col7-<d2v1>-<d3v2>"),
        Bytes.toBytes("col8-<d2v1>-<d3v2>"), Bytes.toBytes("col9-<d2v1>-<d3v2>") };
    for (int i = 0; i < 10; i++) {
      Put put = new Put(ROWS[i]);
      put.setDurability(Durability.SKIP_WAL);
      put.addColumn(FAMILY, QUALIFIERS[i], VALUE);
      ht.put(put);
    }
    Scan scan = new Scan();
    scan.addFamily(FAMILY);
    Filter filter = new KeyOnlyFilter(true);
    scan.setFilter(filter);
    try (ResultScanner scanner = ht.getScanner(scan)) {
      int count = 0;
      for (Result result : scanner) {
        assertEquals(1, result.size());
        assertEquals(Bytes.SIZEOF_INT, result.rawCells()[0].getValueLength());
        assertEquals(VALUE.length, Bytes.toInt(CellUtil.cloneValue(result.rawCells()[0])));
        count++;
      }
      assertEquals(10, count);
    }
  }
}
 
Example #15
Source File: Main.java    From cloud-bigtable-examples with Apache License 2.0 5 votes vote down vote up
@ProcessElement
public void processElement(ProcessContext c) throws IOException {
  Scan scan = new Scan()
      .setRowPrefixFilter(c.element().getBytes())
      .setFilter(new KeyOnlyFilter());

  Table table = getConnection().getTable(TableName.valueOf(tableId));

  for (Result result : table.getScanner(scan)) {
    c.output(result.getRow());
  }
}
 
Example #16
Source File: MultiThreadedClientExample.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public Boolean call() throws Exception {

  // total length in bytes of all read rows.
  int result = 0;

  // Number of rows the scan will read before being considered done.
  int toRead = 100;
  try (Table t = connection.getTable(tableName)) {
    byte[] rk = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
    Scan s = new Scan().withStartRow(rk);

    // This filter will keep the values from being sent accross the wire.
    // This is good for counting or other scans that are checking for
    // existence and don't rely on the value.
    s.setFilter(new KeyOnlyFilter());

    // Don't go back to the server for every single row.
    // We know these rows are small. So ask for 20 at a time.
    // This would be application specific.
    //
    // The goal is to reduce round trips but asking for too
    // many rows can lead to GC problems on client and server sides.
    s.setCaching(20);

    // Don't use the cache. While this is a silly test program it's still good to be
    // explicit that scans normally don't use the block cache.
    s.setCacheBlocks(false);

    // Open up the scanner and close it automatically when done.
    try (ResultScanner rs = t.getScanner(s)) {

      // Now go through rows.
      for (Result r : rs) {
        // Keep track of things size to simulate doing some real work.
        result += r.getRow().length;
        toRead -= 1;

        // Most online applications won't be
        // reading the entire table so this break
        // simulates small to medium size scans,
        // without needing to know an end row.
        if (toRead <= 0)  {
          break;
        }
      }
    }
  }
  return result > 0;
}
 
Example #17
Source File: GridTableHBaseBenchmark.java    From Kylin with Apache License 2.0 4 votes vote down vote up
private static void prepareData(HConnection conn) throws IOException {
    HTableInterface table = conn.getTable(TEST_TABLE);

    try {
        // check how many rows existing
        int nRows = 0;
        Scan scan = new Scan();
        scan.setFilter(new KeyOnlyFilter());
        ResultScanner scanner = table.getScanner(scan);
        for (Result r : scanner) {
            r.getRow(); // nothing to do
            nRows++;
        }

        if (nRows > 0) {
            System.out.println(nRows + " existing rows");
            if (nRows != N_ROWS)
                throw new IOException("Expect " + N_ROWS + " rows but it is not");
            return;
        }

        // insert rows into empty table
        System.out.println("Writing " + N_ROWS + " rows to " + TEST_TABLE);
        long nBytes = 0;
        for (int i = 0; i < N_ROWS; i++) {
            byte[] rowkey = Bytes.toBytes(i);
            Put put = new Put(rowkey);
            byte[] cell = randomBytes();
            put.add(CF, QN, cell);
            table.put(put);
            nBytes += cell.length;
            dot(i, N_ROWS);
        }
        System.out.println();
        System.out.println("Written " + N_ROWS + " rows, " + nBytes + " bytes");

    } finally {
        IOUtils.closeQuietly(table);
    }

}
 
Example #18
Source File: ReadData.java    From java-docs-samples with Apache License 2.0 4 votes vote down vote up
@ProcessElement
public void processElement(PipelineOptions po) {
  // Determine which column will be drawn based on runtime of job.
  long timestampDiff = System.currentTimeMillis() - START_TIME;
  long minutes = (timestampDiff / 1000) / 60;
  int timeOffsetIndex = Math.toIntExact(minutes / KEY_VIZ_WINDOW_MINUTES);

  ReadDataOptions options = po.as(ReadDataOptions.class);
  long count = 0;

  List<RowRange> ranges = getRangesForTimeIndex(timeOffsetIndex, getNumRows(options));
  if (ranges.size() == 0) {
    return;
  }

  try {
    // Scan with a filter that will only return the first key from each row. This filter is used
    // to more efficiently perform row count operations.
    Filter rangeFilters = new MultiRowRangeFilter(ranges);
    FilterList firstKeyFilterWithRanges = new FilterList(
        rangeFilters,
        new FirstKeyOnlyFilter(),
        new KeyOnlyFilter());
    Scan scan =
        new Scan()
            .addFamily(Bytes.toBytes(COLUMN_FAMILY))
            .setFilter(firstKeyFilterWithRanges);

    Table table = getConnection().getTable(TableName.valueOf(options.getBigtableTableId()));
    ResultScanner imageData = table.getScanner(scan);

    // Iterate over stream of rows to count them.
    for (Result row : imageData) {
      count++;
    }
  } catch (Exception e) {
    System.out.println("Error reading.");
    e.printStackTrace();
  }
  System.out.printf("got %d rows\n", count);
}