Java Code Examples for org.apache.hadoop.hbase.client.Scan#addFamily()

The following examples show how to use org.apache.hadoop.hbase.client.Scan#addFamily() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
private void runScanner(Table hTable, int expectedSize, Filter... filters) throws IOException {
  String cf = "f";
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes(cf));
  FilterList filterList = new FilterList(filters);
  scan.setFilter(filterList);

  ResultScanner scanner = hTable.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  Result result;
  long timeBeforeScan = System.currentTimeMillis();
  while ((result = scanner.next()) != null) {
    for (Cell kv : result.listCells()) {
      LOG.info("Got rk: " + Bytes.toStringBinary(CellUtil.cloneRow(kv)) + " cq: "
              + Bytes.toStringBinary(CellUtil.cloneQualifier(kv)));
      results.add(kv);
    }
  }
  long scanTime = System.currentTimeMillis() - timeBeforeScan;
  scanner.close();

  LOG.info("scan time = " + scanTime + "ms");
  LOG.info("found " + results.size() + " results");

  assertEquals(expectedSize, results.size());
}
 
Example 2
@Override
public List<String> getSearchClicksRowKeysWithValidQueryString() {
	LOG.debug("Checking getSearchClicksRowKeys searchclicks table content!");
	Scan scan = new Scan();
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES);
	SingleColumnValueFilter filter = new SingleColumnValueFilter(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES, 
			Bytes.toBytes("querystring"), CompareOp.NOT_EQUAL, Bytes.toBytes("jaiblahblah"));
	filter.setFilterIfMissing(true);
	scan.setFilter(filter);
	List<String> rows = hbaseTemplate.find("searchclicks", scan,
			new RowMapper<String>() {
				@Override
				public String mapRow(Result result, int rowNum)
						throws Exception {
					return new String(result.getRow());
				}
			});
	for (String row : rows) {
		LOG.debug("searchclicks table content, Table returned row key: {}", row);
	}
	LOG.debug("Checking getSearchClicksRowKeys searchclicks table content done!");
	return rows;
}
 
Example 3
Source Project: kylin   File: GridTableHBaseBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
private static void fullScan(Connection conn, boolean[] hits, Stats stats) throws IOException {
    Table table = conn.getTable(TableName.valueOf(TEST_TABLE));
    try {
        stats.markStart();

        Scan scan = new Scan();
        scan.addFamily(CF);
        ResultScanner scanner = table.getScanner(scan);
        int i = 0;
        for (Result r : scanner) {
            if (hits[i])
                stats.consume(r);
            dot(i, N_ROWS);
            i++;
        }

        stats.markEnd();
    } finally {
        IOUtils.closeQuietly(table);
    }
}
 
Example 4
private Scan createScan(Application application, Range range, byte[] family) {
    range = rangeFactory.createStatisticsRange(range);
    if (logger.isDebugEnabled()) {
        logger.debug("scan time:{} ", range.prettyToString());
    }

    // start key is replaced by end key because timestamp has been reversed
    byte[] startKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getTo());
    byte[] endKey = ApplicationMapStatisticsUtils.makeRowKey(application.getName(), application.getServiceTypeCode(), range.getFrom());

    final Scan scan = new Scan();
    scan.setCaching(this.scanCacheSize);
    scan.setStartRow(startKey);
    scan.setStopRow(endKey);
    scan.addFamily(family);
    scan.setId("ApplicationSelfScan");

    return scan;
}
 
Example 5
@Override
public List<String> getSearchClicks() {
	LOG.debug("Checking searchclicks table content!");
	Scan scan = new Scan();
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_CLIENT_BYTES);
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_SEARCH_BYTES);
	scan.addFamily(HbaseJsonEventSerializer.COLUMFAMILY_FILTERS_BYTES);
	List<String> rows = hbaseTemplate.find("searchclicks", scan,
			new RowMapper<String>() {
				@Override
				public String mapRow(Result result, int rowNum)
						throws Exception {
					return Arrays.toString(result.rawCells());
				}
			});
	for (String row : rows) {
		LOG.debug("searchclicks table content, Table returned row: {}", row);
	}
	LOG.debug("Checking searchclicks table content done!");
	return rows;
}
 
Example 6
Source Project: hbase   File: PerformanceEvaluation.java    License: Apache License 2.0 5 votes vote down vote up
@Override
boolean testRow(final int i) throws IOException {
  if (this.testScanner == null) {
    Scan scan =
        new Scan().withStartRow(format(opts.startRow)).setCaching(opts.caching)
            .setCacheBlocks(opts.cacheBlocks).setAsyncPrefetch(opts.asyncPrefetch)
            .setReadType(opts.scanReadType).setScanMetricsEnabled(true);
    for (int family = 0; family < opts.families; family++) {
      byte[] familyName = Bytes.toBytes(FAMILY_NAME_BASE + family);
      if (opts.addColumns) {
        for (int column = 0; column < opts.columns; column++) {
          byte [] qualifier = column == 0? COLUMN_ZERO: Bytes.toBytes("" + column);
          scan.addColumn(familyName, qualifier);
        }
      } else {
        scan.addFamily(familyName);
      }
    }
    if (opts.filterAll) {
      scan.setFilter(new FilterAllFilter());
    }
    this.testScanner = asyncTable.getScanner(scan);
  }
  Result r = testScanner.next();
  updateValueSize(r);
  return true;
}
 
Example 7
Source Project: pinpoint   File: HbaseAgentEventDao.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<AgentEventBo> getAgentEvents(String agentId, Range range, Set<AgentEventType> excludeEventTypes) {
    Objects.requireNonNull(agentId, "agentId");
    Objects.requireNonNull(range, "range");

    Scan scan = new Scan();
    scan.setMaxVersions(1);
    scan.setCaching(SCANNER_CACHE_SIZE);

    scan.setStartRow(createRowKey(agentId, range.getTo()));
    scan.setStopRow(createRowKey(agentId, range.getFrom()));
    scan.addFamily(descriptor.getColumnFamilyName());

    if (!CollectionUtils.isEmpty(excludeEventTypes)) {
        FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        for (AgentEventType excludeEventType : excludeEventTypes) {
            byte[] excludeQualifier = Bytes.toBytes(excludeEventType.getCode());
            filterList.addFilter(new QualifierFilter(CompareFilter.CompareOp.NOT_EQUAL, new BinaryComparator(excludeQualifier)));
        }
        scan.setFilter(filterList);
    }

    TableName agentEventTableName = descriptor.getTableName();
    List<AgentEventBo> agentEvents = this.hbaseOperations2.find(agentEventTableName, scan, agentEventResultsExtractor);
    logger.debug("agentEvents found. {}", agentEvents);
    return agentEvents;
}
 
Example 8
@Override
public void initScan(Scan scan) {
    for (String columnFamily : columnFamilies) {
        scan.addFamily(Bytes.toBytes(columnFamily.trim()));
    }

    super.setMaxVersions(scan);
}
 
Example 9
Source Project: hbase   File: TestSCVFWithMiniCluster.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test the filter by adding all columns of family A in the scan. (OK)
 */
@Test
public void scanWithAllQualifiersOfFamiliyA() throws IOException {
  /* Given */
  Scan scan = new Scan();
  scan.addFamily(FAMILY_A);
  scan.setFilter(scanFilter);

  verify(scan);
}
 
Example 10
Source Project: hbase   File: ThriftHBaseServiceHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int scannerOpenWithPrefix(ByteBuffer tableName,
    ByteBuffer startAndPrefix,
    List<ByteBuffer> columns,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startAndPrefix));
    addAttributes(scan, attributes);
    Filter f = new WhileMatchFilter(
        new PrefixFilter(getBytes(startAndPrefix)));
    scan.setFilter(f);
    if (columns != null && !columns.isEmpty()) {
      for(ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 
Example 11
@Test
public void testReadRequestsCountWithTTLExpiration() throws Exception {
  putTTLExpiredData();

  Scan scan = new Scan();
  scan.addFamily(CF2);
  try (ResultScanner scanner = table.getScanner(scan)) {
    int resultCount = 0;
    for (Result ignore : scanner) {
      resultCount++;
    }
    testReadRequests(resultCount, 2, 1);
  }
}
 
Example 12
private static void projectAllColumnFamilies(PTable table, Scan scan) {
    // Will project all known/declared column families
    scan.getFamilyMap().clear();
    for (PColumnFamily family : table.getColumnFamilies()) {
        scan.addFamily(family.getName().getBytes());
    }
}
 
Example 13
@Override
public ResultScanner getScanner(byte[] family) throws IOException {
  if (tx == null) {
    throw new IOException("Transaction not started");
  }
  Scan scan = new Scan();
  scan.addFamily(family);
  return hTable.getScanner(transactionalizeAction(scan));
}
 
Example 14
Source Project: phoenix   File: ProjectionCompiler.java    License: Apache License 2.0 5 votes vote down vote up
private static void projectAllColumnFamilies(PTable table, Scan scan) {
    // Will project all known/declared column families
    scan.getFamilyMap().clear();
    for (PColumnFamily family : table.getColumnFamilies()) {
        scan.addFamily(family.getName().getBytes());
    }
}
 
Example 15
/**
 * Tests a MR scan using specific start and stop rows.
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void testScan(String start, String stop, String last)
    throws IOException, InterruptedException, ClassNotFoundException {
  String jobName =
      "Scan" + (start != null ? start.toUpperCase(Locale.ROOT) : "Empty") + "To" +
          (stop != null ? stop.toUpperCase(Locale.ROOT) : "Empty");
  LOG.info("Before map/reduce startup - job " + jobName);
  Configuration c = new Configuration(TEST_UTIL.getConfiguration());

  c.set(KEY_STARTROW, start != null ? start : "");
  c.set(KEY_LASTROW, last != null ? last : "");

  List<Scan> scans = new ArrayList<>();

  for (String tableName : TABLES) {
    Scan scan = new Scan();

    scan.addFamily(INPUT_FAMILY);
    scan.setAttribute(Scan.SCAN_ATTRIBUTES_TABLE_NAME, Bytes.toBytes(tableName));

    if (start != null) {
      scan.withStartRow(Bytes.toBytes(start));
    }
    if (stop != null) {
      scan.withStopRow(Bytes.toBytes(stop));
    }

    scans.add(scan);

    LOG.info("scan before: " + scan);
  }

  runJob(jobName, c, scans);
}
 
Example 16
Source Project: phoenix   File: EncodedColumnsUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static void setColumns(PColumn column, PTable table, Scan scan) {
	if (table.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS) {
        // if a table storage scheme is COLUMNS_STORED_IN_SINGLE_CELL set then all columns of a column family are stored in a single cell 
        // (with the qualifier name being same as the family name), just project the column family here
        // so that we can calculate estimatedByteSize correctly in ProjectionCompiler 
		scan.addFamily(column.getFamilyName().getBytes());
	}
    else {
        if (column.getColumnQualifierBytes() != null) {
            scan.addColumn(column.getFamilyName().getBytes(), column.getColumnQualifierBytes());
        }
    }
}
 
Example 17
Source Project: hbase   File: ScannerResultGenerator.java    License: Apache License 2.0 4 votes vote down vote up
public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
  final Filter filter, final int caching ,final boolean cacheBlocks, int limit) throws IOException {
  Table table = RESTServlet.getInstance().getTable(tableName);
  try {
    Scan scan;
    if (rowspec.hasEndRow()) {
      scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
    } else {
      scan = new Scan().withStartRow(rowspec.getStartRow());
    }
    if (rowspec.hasColumns()) {
      byte[][] columns = rowspec.getColumns();
      for (byte[] column: columns) {
        byte[][] split = CellUtil.parseColumn(column);
        if (split.length == 1) {
          scan.addFamily(split[0]);
        } else if (split.length == 2) {
          scan.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid familyAndQualifier provided.");
        }
      }
    }
    scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    scan.readVersions(rowspec.getMaxVersions());
    if (filter != null) {
      scan.setFilter(filter);
    }
    if (caching > 0 ) {
      scan.setCaching(caching);
    }
    if (limit > 0) {
      scan.setLimit(limit);
    }
    scan.setCacheBlocks(cacheBlocks);
    if (rowspec.hasLabels()) {
      scan.setAuthorizations(new Authorizations(rowspec.getLabels()));
    }
    scanner = table.getScanner(scan);
    cached = null;
    id = Long.toString(System.currentTimeMillis()) +
           Integer.toHexString(scanner.hashCode());
  } finally {
    table.close();
  }
}
 
Example 18
public MutationPlan compile(PTable index) throws SQLException {
try (final PhoenixStatement statement = new PhoenixStatement(connection)) {
          String query = "SELECT count(*) FROM " + tableName;
          final QueryPlan plan = statement.compileQuery(query);
          TableRef tableRef = plan.getTableRef();
          Scan scan = plan.getContext().getScan();
          ImmutableBytesWritable ptr = new ImmutableBytesWritable();
          final PTable dataTable = tableRef.getTable();
          List<PTable> indexes = Lists.newArrayListWithExpectedSize(1);
          for (PTable indexTable : dataTable.getIndexes()) {
              if (indexTable.getKey().equals(index.getKey())) {
                  index = indexTable;
                  break;
              }
          }
          // Only build newly created index.
          indexes.add(index);
          IndexMaintainer.serialize(dataTable, ptr, indexes, plan.getContext().getConnection());
          // Set attribute on scan that UngroupedAggregateRegionObserver will switch on.
          // We'll detect that this attribute was set the server-side and write the index
          // rows per region as a result. The value of the attribute will be our persisted
          // index maintainers.
          // Define the LOCAL_INDEX_BUILD as a new static in BaseScannerRegionObserver
          scan.setAttribute(BaseScannerRegionObserver.LOCAL_INDEX_BUILD_PROTO, ByteUtil.copyKeyBytesIfNecessary(ptr));
          // By default, we'd use a FirstKeyOnly filter as nothing else needs to be projected for count(*).
          // However, in this case, we need to project all of the data columns that contribute to the index.
          IndexMaintainer indexMaintainer = index.getIndexMaintainer(dataTable, connection);
          for (ColumnReference columnRef : indexMaintainer.getAllColumns()) {
              if (index.getImmutableStorageScheme() == ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS) {
                  scan.addFamily(columnRef.getFamily());
              } else {
                  scan.addColumn(columnRef.getFamily(), columnRef.getQualifier());
              }
          }
          if (dataTable.isTransactional()) {
              scan.setAttribute(BaseScannerRegionObserver.TX_STATE, connection.getMutationState().encodeTransaction());
          }

          // Go through MutationPlan abstraction so that we can create local indexes
          // with a connectionless connection (which makes testing easier).
          return new PostLocalIndexDDLMutationPlan(plan, dataTable);
  }
}
 
Example 19
Source Project: geowave   File: AggregationEndpoint.java    License: Apache License 2.0 4 votes vote down vote up
private Object getValue(
    final Aggregation aggregation,
    final Filter filter,
    final DataTypeAdapter dataAdapter,
    final Short internalAdapterId,
    final HBaseDistributableFilter hdFilter,
    final boolean blockCaching,
    final int scanCacheSize,
    final String[] authorizations) throws IOException {
  final Scan scan = new Scan();
  scan.setMaxVersions(1);
  scan.setCacheBlocks(blockCaching);

  if (scanCacheSize != HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING) {
    scan.setCaching(scanCacheSize);
  }

  if (filter != null) {
    scan.setFilter(filter);
  }

  if (internalAdapterId != null) {
    scan.addFamily(StringUtils.stringToBinary(ByteArrayUtils.shortToString(internalAdapterId)));
  }

  if (authorizations != null) {
    scan.setAuthorizations(new Authorizations(authorizations));
  }
  env.getRegion().getCoprocessorHost().preScannerOpen(scan);
  try (InternalScanner scanner = env.getRegion().getScanner(scan)) {
    final List<Cell> results = new ArrayList<>();
    boolean hasNext;
    do {
      hasNext = scanner.next(results);
      if (!results.isEmpty()) {
        if (hdFilter != null) {
          if (dataAdapter != null) {
            final Object row = hdFilter.decodeRow(dataAdapter);

            if (row != null) {
              aggregation.aggregate(row);
            } else {
              LOGGER.error("DataAdapter failed to decode row");
            }
          } else {
            aggregation.aggregate(hdFilter.getPersistenceEncoding());
          }
        } else {
          aggregation.aggregate(null);
        }
        results.clear();
      }
    } while (hasNext);
  }
  return aggregation.getResult();
}
 
Example 20
Source Project: phoenix-omid   File: TTable.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Transactional version of {@link Table#getScanner(byte[] family)}
 *
 * @param transaction an instance of transaction to be used
 * @param family      column family
 * @return an instance of ResultScanner
 * @throws IOException if a remote or network exception occurs
 */
public ResultScanner getScanner(Transaction transaction, byte[] family) throws IOException {
    Scan scan = new Scan();
    scan.addFamily(family);
    return getScanner(transaction, scan);
}