Java Code Examples for org.apache.hadoop.hbase.client.Scan#addFamily()

private void runScanner(Table hTable, int expectedSize, Filter... filters) throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes(cf));
  FilterList filterList = new FilterList(filters);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
              + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 

@Override
public List<String> getSearchClicks() {
	LOG.debug("Checking searchclicks table content!");
	Scan scan = new Scan();
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_CLIENT_BYTES);
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES);
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_FILTERS_BYTES);
	List<String> rows = hbaseTemplate.find("searchclicks", scan,
			new RowMapper<String>() {
				@Override
				public String mapRow(Result result, int rowNum)
						throws Exception {
					return Arrays.toString(result.rawCells());
				}
			});
	for (String row : rows) {
		LOG.debug("searchclicks table content, Table returned row: {}", row);
	}
	LOG.debug("Checking searchclicks table content done!");
	return rows;
}
 

private Scan createScan(Application application, Range range, byte[] family) {
    range = rangeFactory.createStatisticsRange(range);
    if (logger.isDebugEnabled()) {
        logger.debug("scan time:{} ", range.prettyToString());
    }

    // start key is replaced by end key because timestamp has been reversed
    byte[] startKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getTo());
    byte[] endKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getFrom());

    final Scan scan = new Scan();
    scan.setCaching(this.scanCacheSize);
    scan.setStartRow(startKey);
    scan.setStopRow(endKey);
    scan.addFamily(family);
    scan.setId("ApplicationSelfScan");

    return scan;
}
 

private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 

@Override
public List<String> getSearchClicksRowKeysWithValidQueryString() {
	LOG.debug("Checking getSearchClicksRowKeys searchclicks table content!");
	Scan scan = new Scan();
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES);
	SingleColumnValueFilter filter = new SingleColumnValueFilter(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES, 
			Bytes.toBytes("querystring"), CompareOp.NOT_EQUAL, Bytes.toBytes("jaiblahblah"));
	filter.setFilterIfMissing(true);
	scan.setFilter(filter);
	List<String> rows = hbaseTemplate.find("searchclicks", scan,
			new RowMapper<String>() {
				@Override
				public String mapRow(Result result, int rowNum)
						throws Exception {
					return new String(result.getRow());
				}
			});
	for (String row : rows) {
		LOG.debug("searchclicks table content, Table returned row key: {}", row);
	}
	LOG.debug("Checking getSearchClicksRowKeys searchclicks table content done!");
	return rows;
}
 

@Override
public void initScan(Scan scan) {
    for (String columnFamily : columnFamilies) {
        scan.addFamily(Bytes.toBytes(columnFamily.trim()));
    }

    super.setMaxVersions(scan);
}
 

/**
 * Test the filter by adding all columns of family A in the scan. (OK)
 */
@Test
public void scanWithAllQualifiersOfFamiliyA() throws IOException {
  /* Given */
  Scan scan = new Scan();
  scan.addFamily(FAMILY_A);
  scan.setFilter(scanFilter);

  verify(scan);
}
 

@Override
public int scannerOpenWithPrefix(ByteBuffer tableName,
    ByteBuffer startAndPrefix,
    List<ByteBuffer> columns,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startAndPrefix));
    addAttributes(scan, attributes);
    Filter f = new WhileMatchFilter(
        new PrefixFilter(getBytes(startAndPrefix)));
    scan.setFilter(f);
    if (columns != null && !columns.isEmpty()) {
      for(ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 

@Test
public void testReadRequestsCountWithTTLExpiration() throws Exception {
  putTTLExpiredData();

  Scan scan = new Scan();
  scan.addFamily(CF2);
  try (ResultScanner scanner = table.getScanner(scan)) {
    int resultCount = 0;
    for (Result ignore : scanner) {
      resultCount++;
    }
    testReadRequests(resultCount, 2, 1);
  }
}
 

private static void projectAllColumnFamilies(PTable table, Scan scan) {
    // Will project all known/declared column families
    scan.getFamilyMap().clear();
    for (PColumnFamily family : table.getColumnFamilies()) {
        scan.addFamily(family.getName().getBytes());
    }
}
 

@Override
public ResultScanner getScanner(byte[] family) throws IOException {
  if (tx == null) {
    throw new IOException("Transaction not started");
  }
  Scan scan = new Scan();
  scan.addFamily(family);
  return hTable.getScanner(transactionalizeAction(scan));
}
 

@Override
public List<AgentEventBo> getAgentEvents(String agentId, Range range, Set<AgentEventType> excludeEventTypes) {
    Objects.requireNonNull(agentId, "agentId");
    Objects.requireNonNull(range, "range");

    Scan scan = new Scan();
    scan.setMaxVersions(1);
    scan.setCaching(SCANNER_CACHE_SIZE);

    scan.setStartRow(createRowKey(agentId, range.getTo()));
    scan.setStopRow(createRowKey(agentId, range.getFrom()));
    scan.addFamily(descriptor.getColumnFamilyName());

    if (!CollectionUtils.isEmpty(excludeEventTypes)) {
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        for (AgentEventType excludeEventType : excludeEventTypes) {
            byte[] excludeQualifier = Bytes.toBytes(excludeEventType.getCode());
            filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(excludeQualifier)));
        }
        scan.setFilter(filterList);
    }

    TableName agentEventTableName = descriptor.getTableName();
    List<AgentEventBo> agentEvents = this.hbaseOperations2.find(agentEventTableName, scan, agentEventResultsExtractor);
    logger.debug("agentEvents found. {}", agentEvents);
    return agentEvents;
}
 

private static void projectAllColumnFamilies(PTable table, Scan scan) {
    // Will project all known/declared column families
    scan.getFamilyMap().clear();
    for (PColumnFamily family : table.getColumnFamilies()) {
        scan.addFamily(family.getName().getBytes());
    }
}
 

@Override
boolean testRow(final int i) throws IOException {
  if (this.testScanner == null) {
    Scan scan =
        new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
            .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
            .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
    for (int family = 0; family < opts.families; family++) {
      byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
      if (opts.addColumns) {
        for (int column = 0; column < opts.columns; column++) {
          byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
          scan.addColumn(familyName, qualifier);
        }
      } else {
        scan.addFamily(familyName);
      }
    }
    if (opts.filterAll) {
      scan.setFilter(new FilterAllFilter());
    }
    this.testScanner = asyncTable.getScanner(scan);
  }
  Result r = testScanner.next();
  updateValueSize(r);
  return true;
}
 

/**
 * Tests a MR scan using specific start and stop rows.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testScan(String start, String stop, String last)
    throws IOException, InterruptedException, ClassNotFoundException {
  String jobName =
      "Scan" + (start != null ? start.toUpperCase(Locale.ROOT) : "Empty") + "To" +
          (stop != null ? stop.toUpperCase(Locale.ROOT) : "Empty");
  LOG.info("Before map/reduce startup - job " + jobName);
  Configuration c = new Configuration(TEST_UTIL.getConfiguration());

  c.set(KEY_STARTROW, start != null ? start : "");
  c.set(KEY_LASTROW, last != null ? last : "");

  List<Scan> scans = new ArrayList<>();

  for (String tableName : TABLES) {
    Scan scan = new Scan();

    scan.addFamily(INPUT_FAMILY);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));

    if (start != null) {
      scan.withStartRow(Bytes.toBytes(start));
    }
    if (stop != null) {
      scan.withStopRow(Bytes.toBytes(stop));
    }

    scans.add(scan);

    LOG.info("scan before: " + scan);
  }

  runJob(jobName, c, scans);
}
 

public static void setColumns(PColumn column, PTable table, Scan scan) {
	if (table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS) {
        // if a table storage scheme is COLUMNS_STORED_IN_SINGLE_CELL set then all columns of a column family are stored in a single cell 
        // (with the qualifier name being same as the family name), just project the column family here
        // so that we can calculate estimatedByteSize correctly in ProjectionCompiler 
		scan.addFamily(column.getFamilyName().getBytes());
	}
    else {
        if (column.getColumnQualifierBytes() != null) {
            scan.addColumn(column.getFamilyName().getBytes(), column.getColumnQualifierBytes());
        }
    }
}
 

public MutationPlan compile(PTable index) throws SQLException {
try (final PhoenixStatement statement = new PhoenixStatement(connection)) {
          String query = "SELECT count(*) FROM " + tableName;
          final QueryPlan plan = statement.compileQuery(query);
          TableRef tableRef = plan.getTableRef();
          Scan scan = plan.getContext().getScan();
          ImmutableBytesWritable ptr = new ImmutableBytesWritable();
          final PTable dataTable = tableRef.getTable();
          List<PTable> indexes = Lists.newArrayListWithExpectedSize(1);
          for (PTable indexTable : dataTable.getIndexes()) {
              if (indexTable.getKey().equals(index.getKey())) {
                  index = indexTable;
                  break;
              }
          }
          // Only build newly created index.
          indexes.add(index);
          IndexMaintainer.serialize(dataTable, ptr, indexes, plan.getContext().getConnection());
          // Set attribute on scan that UngroupedAggregateRegionObserver will switch on.
          // We'll detect that this attribute was set the server-side and write the index
          // rows per region as a result. The value of the attribute will be our persisted
          // index maintainers.
          // Define the LOCAL_INDEX_BUILD as a new static in BaseScannerRegionObserver
          scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, ByteUtil.copyKeyBytesIfNecessary(ptr));
          // By default, we'd use a FirstKeyOnly filter as nothing else needs to be projected for count(*).
          // However, in this case, we need to project all of the data columns that contribute to the index.
          IndexMaintainer indexMaintainer = index.getIndexMaintainer(dataTable, connection);
          for (ColumnReference columnRef : indexMaintainer.getAllColumns()) {
              if (index.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS) {
                  scan.addFamily(columnRef.getFamily());
              } else {
                  scan.addColumn(columnRef.getFamily(), columnRef.getQualifier());
              }
          }
          if (dataTable.isTransactional()) {
              scan.setAttribute(BaseScannerRegionObserver.TX_STATE, connection.getMutationState().encodeTransaction());
          }

          // Go through MutationPlan abstraction so that we can create local indexes
          // with a connectionless connection (which makes testing easier).
          return new PostLocalIndexDDLMutationPlan(plan, dataTable);
  }
}
 

private Object getValue(
    final Aggregation aggregation,
    final Filter filter,
    final DataTypeAdapter dataAdapter,
    final Short internalAdapterId,
    final HBaseDistributableFilter hdFilter,
    final boolean blockCaching,
    final int scanCacheSize,
    final String[] authorizations) throws IOException {
  final Scan scan = new Scan();
  scan.setMaxVersions(1);
  scan.setCacheBlocks(blockCaching);

  if (scanCacheSize != HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING) {
    scan.setCaching(scanCacheSize);
  }

  if (filter != null) {
    scan.setFilter(filter);
  }

  if (internalAdapterId != null) {
    scan.addFamily(StringUtils.stringToBinary(ByteArrayUtils.shortToString(internalAdapterId)));
  }

  if (authorizations != null) {
    scan.setAuthorizations(new Authorizations(authorizations));
  }
  env.getRegion().getCoprocessorHost().preScannerOpen(scan);
  try (InternalScanner scanner = env.getRegion().getScanner(scan)) {
    final List<Cell> results = new ArrayList<>();
    boolean hasNext;
    do {
      hasNext = scanner.next(results);
      if (!results.isEmpty()) {
        if (hdFilter != null) {
          if (dataAdapter != null) {
            final Object row = hdFilter.decodeRow(dataAdapter);

            if (row != null) {
              aggregation.aggregate(row);
            } else {
              LOGGER.error("DataAdapter failed to decode row");
            }
          } else {
            aggregation.aggregate(hdFilter.getPersistenceEncoding());
          }
        } else {
          aggregation.aggregate(null);
        }
        results.clear();
      }
    } while (hasNext);
  }
  return aggregation.getResult();
}
 

public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
  final Filter filter, final int caching ,final boolean cacheBlocks, int limit) throws IOException {
  Table table = RESTServlet.getInstance().getTable(tableName);
  try {
    Scan scan;
    if (rowspec.hasEndRow()) {
      scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
    } else {
      scan = new Scan().withStartRow(rowspec.getStartRow());
    }
    if (rowspec.hasColumns()) {
      byte[][] columns = rowspec.getColumns();
      for (byte[] column: columns) {
        byte[][] split = CellUtil.parseColumn(column);
        if (split.length == 1) {
          scan.addFamily(split[0]);
        } else if (split.length == 2) {
          scan.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid familyAndQualifier provided.");
        }
      }
    }
    scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    scan.readVersions(rowspec.getMaxVersions());
    if (filter != null) {
      scan.setFilter(filter);
    }
    if (caching > 0 ) {
      scan.setCaching(caching);
    }
    if (limit > 0) {
      scan.setLimit(limit);
    }
    scan.setCacheBlocks(cacheBlocks);
    if (rowspec.hasLabels()) {
      scan.setAuthorizations(new Authorizations(rowspec.getLabels()));
    }
    scanner = table.getScanner(scan);
    cached = null;
    id = Long.toString(System.currentTimeMillis()) +
           Integer.toHexString(scanner.hashCode());
  } finally {
    table.close();
  }
}
 

/**
 * Transactional version of {@link Table#getScanner(byte[] family)}
 *
 * @param transaction an instance of transaction to be used
 * @param family      column family
 * @return an instance of ResultScanner
 * @throws IOException if a remote or network exception occurs
 */
public ResultScanner getScanner(Transaction transaction, byte[] family) throws IOException {
    Scan scan = new Scan();
    scan.addFamily(family);
    return getScanner(transaction, scan);
}