Java Code Examples for org.apache.hadoop.hbase.client.Table#getScanner()

The following examples show how to use org.apache.hadoop.hbase.client.Table#getScanner() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: OfflineMetaRebuildTestCore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected void wipeOutMeta() throws IOException {
  // Mess it up by blowing up meta.
  Admin admin = TEST_UTIL.getAdmin();
  Scan s = new Scan();
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  ResultScanner scanner = meta.getScanner(s);
  List<Delete> dels = new ArrayList<>();
  for (Result r : scanner) {
    RegionInfo info =
        CatalogFamilyFormat.getRegionInfo(r);
    if(info != null && !info.getTable().getNamespaceAsString()
        .equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
      Delete d = new Delete(r.getRow());
      dels.add(d);
      admin.unassign(r.getRow(), true);
    }
  }
  meta.delete(dels);
  scanner.close();
  meta.close();
}
 
Example 2
Source File: TestRegionObserverInterface.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testHBASE14489() throws IOException {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Table table = util.createTable(tableName, new byte[][] { A });
  Put put = new Put(ROW);
  put.addColumn(A, A, A);
  table.put(put);

  Scan s = new Scan();
  s.setFilter(new FilterAllFilter());
  ResultScanner scanner = table.getScanner(s);
  try {
    for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
    }
  } finally {
    scanner.close();
  }
  verifyMethodResult(SimpleRegionObserver.class, new String[] { "wasScannerFilterRowCalled" },
    tableName, new Boolean[] { true });
  util.deleteTable(tableName);
  table.close();

}
 
Example 3
Source File: TestUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
public static int getRowCount(Table table, boolean isRaw) throws IOException {
    Scan s = new Scan();
    s.setRaw(isRaw);;
    s.setMaxVersions();
    int rows = 0;
    try (ResultScanner scanner = table.getScanner(s)) {
        Result result = null;
        while ((result = scanner.next()) != null) {
            rows++;
            CellScanner cellScanner = result.cellScanner();
            Cell current = null;
            while (cellScanner.advance()) {
                current = cellScanner.current();
            }
        }
    }
    return rows;
}
 
Example 4
Source File: AclTableMigrationTool.java    From kylin-on-parquet-v2 with Apache License 2.0 6 votes vote down vote up
private void convertToResourceStore(KylinConfig kylinConfig, String tableName, ResourceStore store,
        ResultConverter converter) throws IOException {

    Table table = null;
    ResultScanner rs = null;
    Scan scan = new Scan();
    try {
        table = HBaseConnection.get(kylinConfig.getStorageUrl()).getTable(TableName.valueOf(tableName));
        rs = table.getScanner(scan);
        converter.convertResult(rs, store);
        store.checkAndPutResource(MIGRATE_OK_PREFIX + tableName, new StringEntity(tableName + " migrated"),
                StringEntity.serializer);
    } finally {
        IOUtils.closeQuietly(rs);
        IOUtils.closeQuietly(table);
    }

}
 
Example 5
Source File: GridTableHBaseBenchmark.java    From kylin with Apache License 2.0 6 votes vote down vote up
private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example 6
Source File: TestUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
public static CellCount getCellCount(Table table, boolean isRaw) throws IOException {
    Scan s = new Scan();
    s.setRaw(isRaw);;
    s.setMaxVersions();

    CellCount cellCount = new CellCount();
    try (ResultScanner scanner = table.getScanner(s)) {
        Result result = null;
        while ((result = scanner.next()) != null) {
            CellScanner cellScanner = result.cellScanner();
            Cell current = null;
            while (cellScanner.advance()) {
                current = cellScanner.current();
                cellCount.addCell(Bytes.toString(CellUtil.cloneRow(current)));
            }
        }
    }
    return cellCount;
}
 
Example 7
Source File: PhoenixRuntimeIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private static void assertTenantIds(Expression e, Table htable, Filter filter, String[] tenantIds) throws IOException {
    ImmutableBytesWritable ptr = new ImmutableBytesWritable();
    Scan scan = new Scan();
    scan.setFilter(filter);
    ResultScanner scanner = htable.getScanner(scan);
    Result result = null;
    ResultTuple tuple;
    Set<String> actualTenantIds = Sets.newHashSetWithExpectedSize(tenantIds.length);
    Set<String> expectedTenantIds = new HashSet<>(Arrays.asList(tenantIds));
    while ((result = scanner.next()) != null) {
        tuple = new ResultTuple(result);
        e.evaluate(tuple, ptr);
        String tenantId = (String)PVarchar.INSTANCE.toObject(ptr);
        actualTenantIds.add(tenantId == null ? "" : tenantId);
    }
    assertTrue(actualTenantIds.containsAll(expectedTenantIds));
}
 
Example 8
Source File: HBase_2_ClientService.java    From nifi with Apache License 2.0 6 votes vote down vote up
protected ResultScanner getResults(final Table table, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations) throws IOException {
    final Scan scan = new Scan();
    scan.setStartRow(startRow);
    scan.setStopRow(endRow);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (columns != null && columns.size() > 0) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 
Example 9
Source File: TestMasterTransitions.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static int addToEachStartKey(final int expected) throws IOException {
  Table t = TEST_UTIL.getConnection().getTable(TABLENAME);
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  int rows = 0;
  Scan scan = new Scan();
  scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
  ResultScanner s = meta.getScanner(scan);
  for (Result r = null; (r = s.next()) != null;) {
    RegionInfo hri = CatalogFamilyFormat.getRegionInfo(r);
    if (hri == null) break;
    if (!hri.getTable().equals(TABLENAME)) {
      continue;
    }

    // If start key, add 'aaa'.
    if(!hri.getTable().equals(TABLENAME)) {
      continue;
    }
    byte [] row = getStartKey(hri);
    Put p = new Put(row);
    p.setDurability(Durability.SKIP_WAL);
    p.addColumn(getTestFamily(), getTestQualifier(), row);
    t.put(p);
    rows++;
  }
  s.close();
  Assert.assertEquals(expected, rows);
  t.close();
  meta.close();
  return rows;
}
 
Example 10
Source File: HBase_1_1_2_ClientService.java    From nifi with Apache License 2.0 5 votes vote down vote up
protected ResultScanner getResults(final Table table, final Collection<Column> columns, final Filter filter, final long minTime, List<String> authorizations) throws IOException {
    // Create a new scan. We will set the min timerange as the latest timestamp that
    // we have seen so far. The minimum timestamp is inclusive, so we will get duplicates.
    // We will record any cells that have the latest timestamp, so that when we scan again,
    // we know to throw away those duplicates.
    final Scan scan = new Scan();
    scan.setTimeRange(minTime, Long.MAX_VALUE);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (filter != null) {
        scan.setFilter(filter);
    }

    if (columns != null) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 
Example 11
Source File: IndexToolForNonTxGlobalIndexIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public void deleteAllRows(Connection conn, TableName tableName) throws SQLException,
    IOException {
    Scan scan = new Scan();
    Table table = conn.unwrap(PhoenixConnection.class).getQueryServices().
        getTable(tableName.getName());
    try (ResultScanner scanner = table.getScanner(scan)) {
        for (Result r : scanner) {
            Delete del = new Delete(r.getRow());
            table.delete(del);
        }
    }
}
 
Example 12
Source File: TestFuzzyRowFilterEndToEnd.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void runScanner(Table hTable, int expectedSize, Filter filter1, Filter filter2)
    throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes(cf));
  FilterList filterList = new FilterList(Operator.MUST_PASS_ALL, filter1, filter2);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
          + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 
Example 13
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
void deleteFromScan(Table stateTable, Scan scan) throws IOException {
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    Result next;
    while ((next = scanner.next()) != null) {
      stateTable.delete(new Delete(next.getRow()));
    }
  }
}
 
Example 14
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Nullable
private TimeRegions getNextSetOfTimeRegions(Table stateTable, long time) throws IOException {
  byte[] timeBytes = Bytes.toBytes(getInvertedTime(time));
  Scan scan = new Scan(makeTimeRegionKey(timeBytes, EMPTY_BYTE_ARRAY), REGION_TIME_KEY_PREFIX_STOP);
  scan.addColumn(FAMILY, REGION_TIME_COL);


  long currentRegionTime = -1;
  SortedSet<byte[]> regions = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  Result next;
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    while ((next = scanner.next()) != null) {
      Map.Entry<Long, byte[]> timeRegion = getTimeRegion(next.getRow());
      // Stop if reached next time value
      if (currentRegionTime == -1) {
        currentRegionTime = timeRegion.getKey();
      } else if (timeRegion.getKey() < currentRegionTime) {
        break;
      } else if (timeRegion.getKey() > currentRegionTime) {
        throw new IllegalStateException(
          String.format("Got out of order time %d when expecting time less than or equal to %d",
                        timeRegion.getKey(), currentRegionTime));
      }
      regions.add(timeRegion.getValue());
    }
  }
  return regions.isEmpty() ? null : new TimeRegions(currentRegionTime, Collections.unmodifiableSortedSet(regions));
}
 
Example 15
Source File: TestPartialResultsFromClientSide.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPartialResultWhenRegionMove() throws IOException {
  Table table = createTestTable(TableName.valueOf(name.getMethodName()),
      ROWS, FAMILIES, QUALIFIERS, VALUE);

  moveRegion(table, 1);

  Scan scan = new Scan();
  scan.setMaxResultSize(1);
  scan.setAllowPartialResults(true);
  ResultScanner scanner = table.getScanner(scan);
  for (int i = 0; i < NUM_FAMILIES * NUM_QUALIFIERS - 1; i++) {
    scanner.next();
  }
  Result result1 = scanner.next();
  assertEquals(1, result1.rawCells().length);
  Cell c1 = result1.rawCells()[0];
  assertCell(c1, ROWS[0], FAMILIES[NUM_FAMILIES - 1], QUALIFIERS[NUM_QUALIFIERS - 1]);
  assertFalse(result1.mayHaveMoreCellsInRow());

  moveRegion(table, 2);

  Result result2 = scanner.next();
  assertEquals(1, result2.rawCells().length);
  Cell c2 = result2.rawCells()[0];
  assertCell(c2, ROWS[1], FAMILIES[0], QUALIFIERS[0]);
  assertTrue(result2.mayHaveMoreCellsInRow());

  moveRegion(table, 3);

  Result result3 = scanner.next();
  assertEquals(1, result3.rawCells().length);
  Cell c3 = result3.rawCells()[0];
  assertCell(c3, ROWS[1], FAMILIES[0], QUALIFIERS[1]);
  assertTrue(result3.mayHaveMoreCellsInRow());

}
 
Example 16
Source File: WALReplayWithIndexWritesAndCompressedWALIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private int getKeyValueCount(Table table) throws IOException {
    Scan scan = new Scan();
    scan.setMaxVersions(Integer.MAX_VALUE - 1);

    ResultScanner results = table.getScanner(scan);
    int count = 0;
    for (Result res : results) {
      count += res.listCells().size();
      LOGGER.debug(count + ") " + res);
    }
    results.close();

    return count;
  }
 
Example 17
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
void deleteFromScan(Table stateTable, Scan scan) throws IOException {
  try (ResultScanner scanner = stateTable.getScanner(scan)) {
    Result next;
    while ((next = scanner.next()) != null) {
      stateTable.delete(new Delete(next.getRow()));
    }
  }
}
 
Example 18
Source File: IndexerDryRun.java    From hbase-indexer with Apache License 2.0 4 votes vote down vote up
/**
 * Run a dry run over the configured input, printing SolrInputDocuments to
 * the configured output.
 * 
 * @return 0 if successful, non-zero otherwise
 */
int run() {
  
    long programStartTime = System.currentTimeMillis();
    IndexingSpecification indexingSpec = indexingOpts.getIndexingSpecification();

    IndexerComponentFactory factory = IndexerComponentFactoryUtil.getComponentFactory(indexingSpec.getIndexerComponentFactory(), new ByteArrayInputStream(indexingSpec.getConfiguration()), Maps.<String, String>newHashMap());
    IndexerConf indexerConf = factory.createIndexerConf();

    if (indexerConf.getRowReadMode() != RowReadMode.NEVER) {
        LOG.warn("Changing row read mode from " + indexerConf.getRowReadMode() + " to " + RowReadMode.NEVER);
        indexerConf = new IndexerConfBuilder(indexerConf).rowReadMode(RowReadMode.NEVER).build();
    }

    Map<String, String> params = indexerConf.getGlobalParams();
    if (indexingOpts.morphlineFile != null) {
        params.put(MorphlineResultToSolrMapper.MORPHLINE_FILE_PARAM, indexingOpts.morphlineFile.getPath());
    }
    if (indexingOpts.morphlineId != null) {
        params.put(MorphlineResultToSolrMapper.MORPHLINE_ID_PARAM, indexingOpts.morphlineId);
    }

    for (Map.Entry<String, String> entry : hbaseConf) {
        if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_VARIABLE_PARAM + ".")) {
            params.put(entry.getKey(), entry.getValue());
        }
        if (entry.getKey().startsWith(MorphlineResultToSolrMapper.MORPHLINE_FIELD_PARAM + ".")) {
            params.put(entry.getKey(), entry.getValue());
        }
    }

    MorphlineClasspathUtil.setupJavaCompilerClasspath();
    
    ResultToSolrMapper resultToSolrMapper = ResultToSolrMapperFactory.createResultToSolrMapper(
            indexingSpec.getIndexerName(),
            indexerConf);
    
    Indexer indexer = Indexer.createIndexer(
                            indexingSpec.getIndexerName(),
                            indexerConf,
                            indexingSpec.getTableName(),
                            resultToSolrMapper,
                            null,
                            null,
                            documentWriter);
    
    Scan scan = indexingOpts.getScans().get(0);
    
    Table htable = null;
    Connection connection = null;
    try {
        connection = ConnectionFactory.createConnection(hbaseConf);
        htable = connection.getTable(TableName.valueOf(indexingSpec.getTableName()));
        ResultScanner scanner = htable.getScanner(scan);
        for (Result result : scanner) {
            indexer.indexRowData(ImmutableList.<RowData>of(new ResultWrappingRowData(result,
                    indexingSpec.getTableName().getBytes(Charsets.UTF_8))));
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        Closer.close(htable);
        if (connection != null) {
            try {
                connection.close();
            } catch (IOException ioe) {
                ; // ignore
            }
        }
    }
    
    HBaseMapReduceIndexerTool.goodbye(null, programStartTime);
    return 0;
    
}
 
Example 19
Source File: IntegrationTestBigLinkedList.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public int run(String[] args) throws Exception {
  Options options = new Options();
  options.addOption("s", "start", true, "start key");
  options.addOption("e", "end", true, "end key");
  options.addOption("l", "limit", true, "number to print");

  GnuParser parser = new GnuParser();
  CommandLine cmd = null;
  try {
    cmd = parser.parse(options, args);
    if (cmd.getArgs().length != 0) {
      throw new ParseException("Command takes no arguments");
    }
  } catch (ParseException e) {
    System.err.println("Failed to parse command line " + e.getMessage());
    System.err.println();
    HelpFormatter formatter = new HelpFormatter();
    formatter.printHelp(getClass().getSimpleName(), options);
    System.exit(-1);
  }

  Connection connection = ConnectionFactory.createConnection(getConf());
  Table table = connection.getTable(getTableName(getConf()));

  Scan scan = new Scan();
  scan.setBatch(10000);

  if (cmd.hasOption("s"))
    scan.withStartRow(Bytes.toBytesBinary(cmd.getOptionValue("s")));

  if (cmd.hasOption("e")) {
    scan.withStopRow(Bytes.toBytesBinary(cmd.getOptionValue("e")));
  }

  int limit = 0;
  if (cmd.hasOption("l"))
    limit = Integer.parseInt(cmd.getOptionValue("l"));
  else
    limit = 100;

  ResultScanner scanner = table.getScanner(scan);

  CINode node = new CINode();
  Result result = scanner.next();
  int count = 0;
  while (result != null && count++ < limit) {
    node = getCINode(result, node);
    System.out.printf("%s:%s:%012d:%s\n", Bytes.toStringBinary(node.key),
        Bytes.toStringBinary(node.prev), node.count, node.client);
    result = scanner.next();
  }
  scanner.close();
  table.close();
  connection.close();

  return 0;
}
 
Example 20
Source File: TestFilterWrapper.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testFilterWrapper() {
  int kv_number = 0;
  int row_number = 0;
  try {
    Scan scan = new Scan();
    List<Filter> fs = new ArrayList<>();

    DependentColumnFilter f1 = new DependentColumnFilter(Bytes.toBytes("f1"),
        Bytes.toBytes("c5"), true, CompareOperator.EQUAL,
        new SubstringComparator("c5"));
    PageFilter f2 = new PageFilter(2);
    fs.add(f1);
    fs.add(f2);
    FilterList filter = new FilterList(fs);

    scan.setFilter(filter);
    Table table = connection.getTable(name);
    ResultScanner scanner = table.getScanner(scan);

    // row2 (c1-c4) and row3(c1-c4) are returned
    for (Result result : scanner) {
      row_number++;
      for (Cell kv : result.listCells()) {
        LOG.debug(kv_number + ". kv: " + kv);
        kv_number++;
        assertEquals("Returned row is not correct", Bytes.toString(CellUtil.cloneRow(kv)),
            "row" + ( row_number + 1 ));
      }
    }

    scanner.close();
    table.close();
  } catch (Exception e) {
    // no correct result is expected
    assertNull("Exception happens in scan", e);
  }
  LOG.debug("check the fetched kv number");
  assertEquals("We should get 8 results returned.", 8, kv_number);
  assertEquals("We should get 2 rows returned", 2, row_number);
}