Java Code Examples for org.apache.hadoop.hbase.client.Scan#addColumn()

The following examples show how to use org.apache.hadoop.hbase.client.Scan#addColumn() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
/**
 * Delete prune upper bounds for the regions that are not in the given exclude set, and the
 * prune upper bound is less than the given value.
 * After the invalid list is pruned up to deletionPruneUpperBound, we do not need entries for regions that have
 * prune upper bound less than deletionPruneUpperBound. We however limit the deletion to only regions that are
 * no longer in existence (due to deletion, etc.), to avoid update/delete race conditions.
 *
 * @param deletionPruneUpperBound prune upper bound below which regions will be deleted
 * @param excludeRegions set of regions that should not be deleted
 * @throws IOException when not able to delete data in HBase
 */
public void deletePruneUpperBounds(long deletionPruneUpperBound, SortedSet<byte[]> excludeRegions)
  throws IOException {
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (!excludeRegions.contains(region)) {
          byte[] timeBytes = next.getValue(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (timeBytes != null) {
            long pruneUpperBoundRegion = Bytes.toLong(timeBytes);
            if (pruneUpperBoundRegion < deletionPruneUpperBound) {
              stateTable.delete(new Delete(next.getRow()));
            }
          }
        }
      }
    }
  }
}
 
Example 2
Source File: HBase_2_ClientService.java    From nifi with Apache License 2.0 6 votes vote down vote up
protected ResultScanner getResults(final Table table, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations) throws IOException {
    final Scan scan = new Scan();
    scan.setStartRow(startRow);
    scan.setStopRow(endRow);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (columns != null && columns.size() > 0) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 
Example 3
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
/**
 * Gets a list of {@link RegionPruneInfo} for given regions. Returns all regions if the given regions set is null.
 *
 * @param regions a set of regions
 * @return list of {@link RegionPruneInfo}s.
 * @throws IOException when not able to read the data from HBase
 */
public List<RegionPruneInfo> getPruneInfoForRegions(@Nullable SortedSet<byte[]> regions) throws IOException {
  List<RegionPruneInfo> regionPruneInfos = new ArrayList<>();
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (regions == null || regions.contains(region)) {
          Cell cell = next.getColumnLatestCell(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (cell != null) {
            byte[] pruneUpperBoundBytes = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();
            regionPruneInfos.add(new RegionPruneInfo(region, Bytes.toStringBinary(region),
                                                     Bytes.toLong(pruneUpperBoundBytes), timestamp));
          }
        }
      }
    }
  }
  return Collections.unmodifiableList(regionPruneInfos);
}
 
Example 4
Source File: TestBlocksScanned.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void _testBlocksScanned(TableDescriptor td) throws Exception {
  BlockCache blockCache = BlockCacheFactory.createBlockCache(conf);
  RegionInfo regionInfo =
      RegionInfoBuilder.newBuilder(td.getTableName()).setStartKey(START_KEY).setEndKey(END_KEY)
          .build();
  HRegion r = HBaseTestingUtility.createRegionAndWAL(regionInfo, testDir, conf, td, blockCache);
  addContent(r, FAMILY, COL);
  r.flush(true);

  CacheStats stats = blockCache.getStats();
  long before = stats.getHitCount() + stats.getMissCount();
  // Do simple test of getting one row only first.
  Scan scan = new Scan().withStartRow(Bytes.toBytes("aaa")).withStopRow(Bytes.toBytes("aaz"))
      .setReadType(Scan.ReadType.PREAD);
  scan.addColumn(FAMILY, COL);
  scan.readVersions(1);

  InternalScanner s = r.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  while (s.next(results));
  s.close();

  int expectResultSize = 'z' - 'a';
  assertEquals(expectResultSize, results.size());

  int kvPerBlock = (int) Math.ceil(BLOCK_SIZE /
      (double) KeyValueUtil.ensureKeyValue(results.get(0)).getLength());
  assertEquals(2, kvPerBlock);

  long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
  long expectIndexBlockRead = expectDataBlockRead;

  assertEquals(expectIndexBlockRead + expectDataBlockRead,
      stats.getHitCount() + stats.getMissCount() - before);
}
 
Example 5
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
/**
 * Delete all inactive transaction bounds recorded for a time less than the given time
 *
 * @param time time in milliseconds
 * @throws IOException when not able to delete data in HBase
 */
public void deleteInactiveTransactionBoundsOnOrBeforeTime(long time) throws IOException {
  try (HTableInterface stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan(makeInactiveTransactionBoundTimeKey(Bytes.toBytes(getInvertedTime(time))),
                         INACTIVE_TRANSACTION_BOUND_TIME_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, INACTIVE_TRANSACTION_BOUND_TIME_COL);
    deleteFromScan(stateTable, scan);
  }
}
 
Example 6
Source File: PerformanceEvaluation.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
void testRow(final int i) throws IOException {
  if (this.testScanner == null) {
    Scan scan = new Scan().withStartRow(format(this.startRow));
    scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
    this.testScanner = table.getScanner(scan);
  }
  testScanner.next();
}
 
Example 7
Source File: DefaultVisibilityLabelServiceImpl.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public List<String> getUserAuths(byte[] user, boolean systemCall)
    throws IOException {
  assert (labelsRegion != null || systemCall);
  if (systemCall || labelsRegion == null) {
    return this.labelsCache.getUserAuths(Bytes.toString(user));
  }
  Scan s = new Scan();
  if (user != null && user.length > 0) {
    s.addColumn(LABELS_TABLE_FAMILY, user);
  }
  Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
      new Authorizations(SYSTEM_LABEL));
  s.setFilter(filter);
  ArrayList<String> auths = new ArrayList<>();
  RegionScanner scanner = this.labelsRegion.getScanner(s);
  try {
    List<Cell> results = new ArrayList<>(1);
    while (true) {
      scanner.next(results);
      if (results.isEmpty()) break;
      Cell cell = results.get(0);
      int ordinal = PrivateCellUtil.getRowAsInt(cell);
      String label = this.labelsCache.getLabel(ordinal);
      if (label != null) {
        auths.add(label);
      }
      results.clear();
    }
  } finally {
    scanner.close();
  }
  return auths;
}
 
Example 8
Source File: UserService.java    From Kylin with Apache License 2.0 5 votes vote down vote up
@Override
public List<String> getUserAuthorities() {
    Scan s = new Scan();
    s.addColumn(Bytes.toBytes(USER_AUTHORITY_FAMILY), Bytes.toBytes(USER_AUTHORITY_COLUMN));

    List<String> authorities = new ArrayList<String>();
    HTableInterface htable = null;
    ResultScanner scanner = null;
    try {
        htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
        scanner = htable.getScanner(s);

        for (Result result = scanner.next(); result != null; result = scanner.next()) {
            byte[] uaBytes = result.getValue(Bytes.toBytes(USER_AUTHORITY_FAMILY), Bytes.toBytes(USER_AUTHORITY_COLUMN));
            Collection<? extends GrantedAuthority> authCollection = Arrays.asList(ugaSerializer.deserialize(uaBytes));

            for (GrantedAuthority auth : authCollection) {
                if (!authorities.contains(auth.getAuthority())) {
                    authorities.add(auth.getAuthority());
                }
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e.getMessage(), e);
    } finally {
        IOUtils.closeQuietly(scanner);
        IOUtils.closeQuietly(htable);
    }

    return authorities;
}
 
Example 9
Source File: TransactionAwareHTable.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  if (tx == null) {
    throw new IOException("Transaction not started");
  }
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return hTable.getScanner(transactionalizeAction(scan));
}
 
Example 10
Source File: ThriftHBaseServiceHandler.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow,
    List<ByteBuffer> columns, long timestamp,
    Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startRow));
    addAttributes(scan, attributes);
    scan.setTimeRange(0, timestamp);
    if (columns != null && !columns.isEmpty()) {
      for (ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 
Example 11
Source File: DataJanitorState.java    From phoenix-tephra with Apache License 2.0 5 votes vote down vote up
/**
 * Delete empty region records saved on or before the given time.
 *
 * @param time time in milliseconds
 */
public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException {
  try (Table stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan();
    scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY));
    scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL);
    deleteFromScan(stateTable, scan);
  }
}
 
Example 12
Source File: ThriftHBaseServiceHandler.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public int scannerOpenWithPrefix(ByteBuffer tableName,
    ByteBuffer startAndPrefix,
    List<ByteBuffer> columns,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startAndPrefix));
    addAttributes(scan, attributes);
    Filter f = new WhileMatchFilter(
        new PrefixFilter(getBytes(startAndPrefix)));
    scan.setFilter(f);
    if (columns != null && !columns.isEmpty()) {
      for(ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 
Example 13
Source File: HBaseRowInputFormat.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Override
protected Scan getScanner() {
	Scan scan = new Scan();
	for (int f = 0; f < families.length; f++) {
		byte[] family = families[f];
		for (int q = 0; q < qualifiers[f].length; q++) {
			byte[] quantifier = qualifiers[f][q];
			scan.addColumn(family, quantifier);
		}
	}
	return scan;
}
 
Example 14
Source File: ThriftTable.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return getScanner(scan);
}
 
Example 15
Source File: ScanModifyingObserver.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void preScannerOpen(
    ObserverContext<RegionCoprocessorEnvironment> c, Scan scan) throws IOException {
  // Add another family:qualifier
  scan.addColumn(FAMILY_TO_ADD, QUALIFIER_TO_ADD);
}
 
Example 16
Source File: RemoteHTable.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return new Scanner(scan);
}
 
Example 17
Source File: TestBaillisAnomaliesWithTXs.java    From phoenix-omid with Apache License 2.0 4 votes vote down vote up
@Test(timeOut = 10_000)
public void testSIDoesNotPreventWriteSkew(ITestContext context) throws Exception {
    // TX History for G2-item:
    // begin; set transaction isolation level repeatable read; -- T1
    // begin; set transaction isolation level repeatable read; -- T2
    // select * from test where id in (1,2); -- T1
    // select * from test where id in (1,2); -- T2
    // update test set value = 11 where id = 1; -- T1
    // update test set value = 21 where id = 2; -- T2
    // commit; -- T1
    // commit; -- T2

    // 0) Start transactions
    TransactionManager tm = newTransactionManager(context);
    TTable txTable = new TTable(connection, TEST_TABLE);
    Transaction tx1 = tm.begin();
    Transaction tx2 = tm.begin();

    Scan rowId12Scan = new Scan(rowId1, rowId3);
    rowId12Scan.addColumn(famName, colName);

    // 1) select * from test where id in (1,2); -- T1
    ResultScanner tx1Scanner = txTable.getScanner(tx1, rowId12Scan);
    Result res = tx1Scanner.next();
    int count = 0;
    while (res != null) {
        LOG.info("RESSS {}", res);
        LOG.info("Row id {} with value {}", Bytes.toString(res.getRow()), Bytes.toInt(res.getValue(famName, colName)));
        switch (count) {
            case 0:
                assertEquals(res.getRow(), rowId1);
                assertEquals(res.getValue(famName, colName), dataValue1);
                break;
            case 1:
                assertEquals(res.getRow(), rowId2);
                assertEquals(res.getValue(famName, colName), dataValue2);
                break;
            default:
                fail();
        }
        res = tx1Scanner.next();
        count++;
    }
    assertEquals(count, 2);

    // 2) select * from test where id in (1,2); -- T2
    ResultScanner tx2Scanner = txTable.getScanner(tx1, rowId12Scan);
    res = tx2Scanner.next();
    count = 0;
    while (res != null) {
        LOG.info("RESSS {}", res);
        LOG.info("Row id {} with value {}", Bytes.toString(res.getRow()), Bytes.toInt(res.getValue(famName, colName)));
        switch (count) {
            case 0:
                assertEquals(res.getRow(), rowId1);
                assertEquals(res.getValue(famName, colName), dataValue1);
                break;
            case 1:
                assertEquals(res.getRow(), rowId2);
                assertEquals(res.getValue(famName, colName), dataValue2);
                break;
            default:
                fail();
        }
        res = tx2Scanner.next();
        count++;
    }
    assertEquals(count, 2);

    // 3) update test set value = 11 where id = 1; -- T1
    Put updateRow1Tx1 = new Put(rowId1);
    updateRow1Tx1.addColumn(famName, colName, Bytes.toBytes("11"));
    txTable.put(tx1, updateRow1Tx1);

    // 4) update test set value = 21 where id = 2; -- T2
    Put updateRow2Tx2 = new Put(rowId2);
    updateRow2Tx2.addColumn(famName, colName, Bytes.toBytes("21"));
    txTable.put(tx2, updateRow2Tx2);

    // 5) commit; -- T1
    tm.commit(tx1);

    // 6) commit; -- T2
    tm.commit(tx2);
}
 
Example 18
Source File: ThriftHBaseServiceHandler.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan();
    addAttributes(scan, attributes);
    if (tScan.isSetStartRow()) {
      scan.withStartRow(tScan.getStartRow());
    }
    if (tScan.isSetStopRow()) {
      scan.withStopRow(tScan.getStopRow());
    }
    if (tScan.isSetTimestamp()) {
      scan.setTimeRange(0, tScan.getTimestamp());
    }
    if (tScan.isSetCaching()) {
      scan.setCaching(tScan.getCaching());
    }
    if (tScan.isSetBatchSize()) {
      scan.setBatch(tScan.getBatchSize());
    }
    if (tScan.isSetColumns() && !tScan.getColumns().isEmpty()) {
      for(ByteBuffer column : tScan.getColumns()) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    if (tScan.isSetFilterString()) {
      ParseFilter parseFilter = new ParseFilter();
      scan.setFilter(
          parseFilter.parseFilterString(tScan.getFilterString()));
    }
    if (tScan.isSetReversed()) {
      scan.setReversed(tScan.isReversed());
    }
    if (tScan.isSetCacheBlocks()) {
      scan.setCacheBlocks(tScan.isCacheBlocks());
    }
    return addScanner(table.getScanner(scan), tScan.sortColumns);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 
Example 19
Source File: RowCounter.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Sets up the actual job.
 *
 * @param conf  The current configuration.
 * @param args  The command line parameters.
 * @return The newly created job.
 * @throws IOException When setting up the job fails.
 * @deprecated as of release 2.3.0. Will be removed on 4.0.0. Please use main method instead.
 */
@Deprecated
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
  String tableName = args[0];
  List<MultiRowRangeFilter.RowRange> rowRangeList = null;
  long startTime = 0;
  long endTime = 0;

  StringBuilder sb = new StringBuilder();

  final String rangeSwitch = "--range=";
  final String startTimeArgKey = "--starttime=";
  final String endTimeArgKey = "--endtime=";
  final String expectedCountArg = "--expected-count=";

  // First argument is table name, starting from second
  for (int i = 1; i < args.length; i++) {
    if (args[i].startsWith(rangeSwitch)) {
      try {
        rowRangeList = parseRowRangeParameter(
          args[i].substring(args[1].indexOf(rangeSwitch)+rangeSwitch.length()));
      } catch (IllegalArgumentException e) {
        return null;
      }
      continue;
    }
    if (args[i].startsWith(startTimeArgKey)) {
      startTime = Long.parseLong(args[i].substring(startTimeArgKey.length()));
      continue;
    }
    if (args[i].startsWith(endTimeArgKey)) {
      endTime = Long.parseLong(args[i].substring(endTimeArgKey.length()));
      continue;
    }
    if (args[i].startsWith(expectedCountArg)) {
      conf.setLong(EXPECTED_COUNT_KEY,
        Long.parseLong(args[i].substring(expectedCountArg.length())));
      continue;
    }
    // if no switch, assume column names
    sb.append(args[i]);
    sb.append(" ");
  }
  if (endTime < startTime) {
    printUsage("--endtime=" + endTime + " needs to be greater than --starttime=" + startTime);
    return null;
  }

  Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
  job.setJarByClass(RowCounter.class);
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  setScanFilter(scan, rowRangeList);
  if (sb.length() > 0) {
    for (String columnName : sb.toString().trim().split(" ")) {
      String family = StringUtils.substringBefore(columnName, ":");
      String qualifier = StringUtils.substringAfter(columnName, ":");

      if (StringUtils.isBlank(qualifier)) {
        scan.addFamily(Bytes.toBytes(family));
      }
      else {
        scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
      }
    }
  }
  scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
  job.setOutputFormatClass(NullOutputFormat.class);
  TableMapReduceUtil.initTableMapperJob(tableName, scan,
    RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
  job.setNumReduceTasks(0);
  return job;
}
 
Example 20
Source File: TestScanWithBloomError.java    From hbase with Apache License 2.0 4 votes vote down vote up
private void addColumnSetToScan(Scan scan, int[] colIds) {
  for (int colId : colIds) {
    scan.addColumn(FAMILY_BYTES,
        Bytes.toBytes(qualFromId(colId)));
  }
}