Java Code Examples for org.apache.hadoop.hbase.client.Scan#addColumn()

/**
 * Delete prune upper bounds for the regions that are not in the given exclude set, and the
 * prune upper bound is less than the given value.
 * After the invalid list is pruned up to deletionPruneUpperBound, we do not need entries for regions that have
 * prune upper bound less than deletionPruneUpperBound. We however limit the deletion to only regions that are
 * no longer in existence (due to deletion, etc.), to avoid update/delete race conditions.
 *
 * @param deletionPruneUpperBound prune upper bound below which regions will be deleted
 * @param excludeRegions set of regions that should not be deleted
 * @throws IOException when not able to delete data in HBase
 */
public void deletePruneUpperBounds(long deletionPruneUpperBound, SortedSet<byte[]> excludeRegions)
  throws IOException {
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (!excludeRegions.contains(region)) {
          byte[] timeBytes = next.getValue(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (timeBytes != null) {
            long pruneUpperBoundRegion = Bytes.toLong(timeBytes);
            if (pruneUpperBoundRegion < deletionPruneUpperBound) {
              stateTable.delete(new Delete(next.getRow()));
            }
          }
        }
      }
    }
  }
}
 

protected ResultScanner getResults(final Table table, final byte[] startRow, final byte[] endRow, final Collection<Column> columns, List<String> authorizations) throws IOException {
    final Scan scan = new Scan();
    scan.setStartRow(startRow);
    scan.setStopRow(endRow);

    if (authorizations != null && authorizations.size() > 0) {
        scan.setAuthorizations(new Authorizations(authorizations));
    }

    if (columns != null && columns.size() > 0) {
        for (Column col : columns) {
            if (col.getQualifier() == null) {
                scan.addFamily(col.getFamily());
            } else {
                scan.addColumn(col.getFamily(), col.getQualifier());
            }
        }
    }

    return table.getScanner(scan);
}
 

/**
 * Gets a list of {@link RegionPruneInfo} for given regions. Returns all regions if the given regions set is null.
 *
 * @param regions a set of regions
 * @return list of {@link RegionPruneInfo}s.
 * @throws IOException when not able to read the data from HBase
 */
public List<RegionPruneInfo> getPruneInfoForRegions(@Nullable SortedSet<byte[]> regions) throws IOException {
  List<RegionPruneInfo> regionPruneInfos = new ArrayList<>();
  try (Table stateTable = stateTableSupplier.get()) {
    byte[] startRow = makeRegionKey(EMPTY_BYTE_ARRAY);
    Scan scan = new Scan(startRow, REGION_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, PRUNE_UPPER_BOUND_COL);

    try (ResultScanner scanner = stateTable.getScanner(scan)) {
      Result next;
      while ((next = scanner.next()) != null) {
        byte[] region = getRegionFromKey(next.getRow());
        if (regions == null || regions.contains(region)) {
          Cell cell = next.getColumnLatestCell(FAMILY, PRUNE_UPPER_BOUND_COL);
          if (cell != null) {
            byte[] pruneUpperBoundBytes = CellUtil.cloneValue(cell);
            long timestamp = cell.getTimestamp();
            regionPruneInfos.add(new RegionPruneInfo(region, Bytes.toStringBinary(region),
                                                     Bytes.toLong(pruneUpperBoundBytes), timestamp));
          }
        }
      }
    }
  }
  return Collections.unmodifiableList(regionPruneInfos);
}
 

private void _testBlocksScanned(TableDescriptor td) throws Exception {
  BlockCache blockCache = BlockCacheFactory.createBlockCache(conf);
  RegionInfo regionInfo =
      RegionInfoBuilder.newBuilder(td.getTableName()).setStartKey(START_KEY).setEndKey(END_KEY)
          .build();
  HRegion r = HBaseTestingUtility.createRegionAndWAL(regionInfo, testDir, conf, td, blockCache);
  addContent(r, FAMILY, COL);
  r.flush(true);

  CacheStats stats = blockCache.getStats();
  long before = stats.getHitCount() + stats.getMissCount();
  // Do simple test of getting one row only first.
  Scan scan = new Scan().withStartRow(Bytes.toBytes("aaa")).withStopRow(Bytes.toBytes("aaz"))
      .setReadType(Scan.ReadType.PREAD);
  scan.addColumn(FAMILY, COL);
  scan.readVersions(1);

  InternalScanner s = r.getScanner(scan);
  List<Cell> results = new ArrayList<>();
  while (s.next(results));
  s.close();

  int expectResultSize = 'z' - 'a';
  assertEquals(expectResultSize, results.size());

  int kvPerBlock = (int) Math.ceil(BLOCK_SIZE /
      (double) KeyValueUtil.ensureKeyValue(results.get(0)).getLength());
  assertEquals(2, kvPerBlock);

  long expectDataBlockRead = (long) Math.ceil(expectResultSize / (double) kvPerBlock);
  long expectIndexBlockRead = expectDataBlockRead;

  assertEquals(expectIndexBlockRead + expectDataBlockRead,
      stats.getHitCount() + stats.getMissCount() - before);
}
 

/**
 * Delete all inactive transaction bounds recorded for a time less than the given time
 *
 * @param time time in milliseconds
 * @throws IOException when not able to delete data in HBase
 */
public void deleteInactiveTransactionBoundsOnOrBeforeTime(long time) throws IOException {
  try (HTableInterface stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan(makeInactiveTransactionBoundTimeKey(Bytes.toBytes(getInvertedTime(time))),
                         INACTIVE_TRANSACTION_BOUND_TIME_KEY_PREFIX_STOP);
    scan.addColumn(FAMILY, INACTIVE_TRANSACTION_BOUND_TIME_COL);
    deleteFromScan(stateTable, scan);
  }
}
 

@Override
void testRow(final int i) throws IOException {
  if (this.testScanner == null) {
    Scan scan = new Scan().withStartRow(format(this.startRow));
    scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
    this.testScanner = table.getScanner(scan);
  }
  testScanner.next();
}
 

@Override
public List<String> getUserAuths(byte[] user, boolean systemCall)
    throws IOException {
  assert (labelsRegion != null || systemCall);
  if (systemCall || labelsRegion == null) {
    return this.labelsCache.getUserAuths(Bytes.toString(user));
  }
  Scan s = new Scan();
  if (user != null && user.length > 0) {
    s.addColumn(LABELS_TABLE_FAMILY, user);
  }
  Filter filter = VisibilityUtils.createVisibilityLabelFilter(this.labelsRegion,
      new Authorizations(SYSTEM_LABEL));
  s.setFilter(filter);
  ArrayList<String> auths = new ArrayList<>();
  RegionScanner scanner = this.labelsRegion.getScanner(s);
  try {
    List<Cell> results = new ArrayList<>(1);
    while (true) {
      scanner.next(results);
      if (results.isEmpty()) break;
      Cell cell = results.get(0);
      int ordinal = PrivateCellUtil.getRowAsInt(cell);
      String label = this.labelsCache.getLabel(ordinal);
      if (label != null) {
        auths.add(label);
      }
      results.clear();
    }
  } finally {
    scanner.close();
  }
  return auths;
}
 

@Override
public List<String> getUserAuthorities() {
    Scan s = new Scan();
    s.addColumn(Bytes.toBytes(USER_AUTHORITY_FAMILY), Bytes.toBytes(USER_AUTHORITY_COLUMN));

    List<String> authorities = new ArrayList<String>();
    HTableInterface htable = null;
    ResultScanner scanner = null;
    try {
        htable = HBaseConnection.get(hbaseUrl).getTable(userTableName);
        scanner = htable.getScanner(s);

        for (Result result = scanner.next(); result != null; result = scanner.next()) {
            byte[] uaBytes = result.getValue(Bytes.toBytes(USER_AUTHORITY_FAMILY), Bytes.toBytes(USER_AUTHORITY_COLUMN));
            Collection<? extends GrantedAuthority> authCollection = Arrays.asList(ugaSerializer.deserialize(uaBytes));

            for (GrantedAuthority auth : authCollection) {
                if (!authorities.contains(auth.getAuthority())) {
                    authorities.add(auth.getAuthority());
                }
            }
        }
    } catch (IOException e) {
        throw new RuntimeException(e.getMessage(), e);
    } finally {
        IOUtils.closeQuietly(scanner);
        IOUtils.closeQuietly(htable);
    }

    return authorities;
}
 

@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  if (tx == null) {
    throw new IOException("Transaction not started");
  }
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return hTable.getScanner(transactionalizeAction(scan));
}
 

@Override
public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow,
    List<ByteBuffer> columns, long timestamp,
    Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startRow));
    addAttributes(scan, attributes);
    scan.setTimeRange(0, timestamp);
    if (columns != null && !columns.isEmpty()) {
      for (ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 

/**
 * Delete empty region records saved on or before the given time.
 *
 * @param time time in milliseconds
 */
public void deleteEmptyRegionsOnOrBeforeTime(long time) throws IOException {
  try (Table stateTable = stateTableSupplier.get()) {
    Scan scan = new Scan();
    scan.setStopRow(makeEmptyRegionTimeKey(Bytes.toBytes(time + 1), EMPTY_BYTE_ARRAY));
    scan.addColumn(FAMILY, EMPTY_REGION_TIME_COL);
    deleteFromScan(stateTable, scan);
  }
}
 

@Override
public int scannerOpenWithPrefix(ByteBuffer tableName,
    ByteBuffer startAndPrefix,
    List<ByteBuffer> columns,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan().withStartRow(getBytes(startAndPrefix));
    addAttributes(scan, attributes);
    Filter f = new WhileMatchFilter(
        new PrefixFilter(getBytes(startAndPrefix)));
    scan.setFilter(f);
    if (columns != null && !columns.isEmpty()) {
      for(ByteBuffer column : columns) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 

@Override
protected Scan getScanner() {
	Scan scan = new Scan();
	for (int f = 0; f < families.length; f++) {
		byte[] family = families[f];
		for (int q = 0; q < qualifiers[f].length; q++) {
			byte[] quantifier = qualifiers[f][q];
			scan.addColumn(family, quantifier);
		}
	}
	return scan;
}
 

@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return getScanner(scan);
}
 

@Override
public void preScannerOpen(
    ObserverContext<RegionCoprocessorEnvironment> c, Scan scan) throws IOException {
  // Add another family:qualifier
  scan.addColumn(FAMILY_TO_ADD, QUALIFIER_TO_ADD);
}
 

@Override
public ResultScanner getScanner(byte[] family, byte[] qualifier) throws IOException {
  Scan scan = new Scan();
  scan.addColumn(family, qualifier);
  return new Scanner(scan);
}
 

@Test(timeOut = 10_000)
public void testSIDoesNotPreventWriteSkew(ITestContext context) throws Exception {
    // TX History for G2-item:
    // begin; set transaction isolation level repeatable read; -- T1
    // begin; set transaction isolation level repeatable read; -- T2
    // select * from test where id in (1,2); -- T1
    // select * from test where id in (1,2); -- T2
    // update test set value = 11 where id = 1; -- T1
    // update test set value = 21 where id = 2; -- T2
    // commit; -- T1
    // commit; -- T2

    // 0) Start transactions
    TransactionManager tm = newTransactionManager(context);
    TTable txTable = new TTable(connection, TEST_TABLE);
    Transaction tx1 = tm.begin();
    Transaction tx2 = tm.begin();

    Scan rowId12Scan = new Scan(rowId1, rowId3);
    rowId12Scan.addColumn(famName, colName);

    // 1) select * from test where id in (1,2); -- T1
    ResultScanner tx1Scanner = txTable.getScanner(tx1, rowId12Scan);
    Result res = tx1Scanner.next();
    int count = 0;
    while (res != null) {
        LOG.info("RESSS {}", res);
        LOG.info("Row id {} with value {}", Bytes.toString(res.getRow()), Bytes.toInt(res.getValue(famName, colName)));
        switch (count) {
            case 0:
                assertEquals(res.getRow(), rowId1);
                assertEquals(res.getValue(famName, colName), dataValue1);
                break;
            case 1:
                assertEquals(res.getRow(), rowId2);
                assertEquals(res.getValue(famName, colName), dataValue2);
                break;
            default:
                fail();
        }
        res = tx1Scanner.next();
        count++;
    }
    assertEquals(count, 2);

    // 2) select * from test where id in (1,2); -- T2
    ResultScanner tx2Scanner = txTable.getScanner(tx1, rowId12Scan);
    res = tx2Scanner.next();
    count = 0;
    while (res != null) {
        LOG.info("RESSS {}", res);
        LOG.info("Row id {} with value {}", Bytes.toString(res.getRow()), Bytes.toInt(res.getValue(famName, colName)));
        switch (count) {
            case 0:
                assertEquals(res.getRow(), rowId1);
                assertEquals(res.getValue(famName, colName), dataValue1);
                break;
            case 1:
                assertEquals(res.getRow(), rowId2);
                assertEquals(res.getValue(famName, colName), dataValue2);
                break;
            default:
                fail();
        }
        res = tx2Scanner.next();
        count++;
    }
    assertEquals(count, 2);

    // 3) update test set value = 11 where id = 1; -- T1
    Put updateRow1Tx1 = new Put(rowId1);
    updateRow1Tx1.addColumn(famName, colName, Bytes.toBytes("11"));
    txTable.put(tx1, updateRow1Tx1);

    // 4) update test set value = 21 where id = 2; -- T2
    Put updateRow2Tx2 = new Put(rowId2);
    updateRow2Tx2.addColumn(famName, colName, Bytes.toBytes("21"));
    txTable.put(tx2, updateRow2Tx2);

    // 5) commit; -- T1
    tm.commit(tx1);

    // 6) commit; -- T2
    tm.commit(tx2);
}
 

@Override
public int scannerOpenWithScan(ByteBuffer tableName, TScan tScan,
    Map<ByteBuffer, ByteBuffer> attributes)
    throws IOError {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan();
    addAttributes(scan, attributes);
    if (tScan.isSetStartRow()) {
      scan.withStartRow(tScan.getStartRow());
    }
    if (tScan.isSetStopRow()) {
      scan.withStopRow(tScan.getStopRow());
    }
    if (tScan.isSetTimestamp()) {
      scan.setTimeRange(0, tScan.getTimestamp());
    }
    if (tScan.isSetCaching()) {
      scan.setCaching(tScan.getCaching());
    }
    if (tScan.isSetBatchSize()) {
      scan.setBatch(tScan.getBatchSize());
    }
    if (tScan.isSetColumns() && !tScan.getColumns().isEmpty()) {
      for(ByteBuffer column : tScan.getColumns()) {
        byte [][] famQf = CellUtil.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    if (tScan.isSetFilterString()) {
      ParseFilter parseFilter = new ParseFilter();
      scan.setFilter(
          parseFilter.parseFilterString(tScan.getFilterString()));
    }
    if (tScan.isSetReversed()) {
      scan.setReversed(tScan.isReversed());
    }
    if (tScan.isSetCacheBlocks()) {
      scan.setCacheBlocks(tScan.isCacheBlocks());
    }
    return addScanner(table.getScanner(scan), tScan.sortColumns);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw getIOError(e);
  } finally{
    closeTable(table);
  }
}
 

/**
 * Sets up the actual job.
 *
 * @param conf  The current configuration.
 * @param args  The command line parameters.
 * @return The newly created job.
 * @throws IOException When setting up the job fails.
 * @deprecated as of release 2.3.0. Will be removed on 4.0.0. Please use main method instead.
 */
@Deprecated
public static Job createSubmittableJob(Configuration conf, String[] args)
  throws IOException {
  String tableName = args[0];
  List<MultiRowRangeFilter.RowRange> rowRangeList = null;
  long startTime = 0;
  long endTime = 0;

  StringBuilder sb = new StringBuilder();

  final String rangeSwitch = "--range=";
  final String startTimeArgKey = "--starttime=";
  final String endTimeArgKey = "--endtime=";
  final String expectedCountArg = "--expected-count=";

  // First argument is table name, starting from second
  for (int i = 1; i < args.length; i++) {
    if (args[i].startsWith(rangeSwitch)) {
      try {
        rowRangeList = parseRowRangeParameter(
          args[i].substring(args[1].indexOf(rangeSwitch)+rangeSwitch.length()));
      } catch (IllegalArgumentException e) {
        return null;
      }
      continue;
    }
    if (args[i].startsWith(startTimeArgKey)) {
      startTime = Long.parseLong(args[i].substring(startTimeArgKey.length()));
      continue;
    }
    if (args[i].startsWith(endTimeArgKey)) {
      endTime = Long.parseLong(args[i].substring(endTimeArgKey.length()));
      continue;
    }
    if (args[i].startsWith(expectedCountArg)) {
      conf.setLong(EXPECTED_COUNT_KEY,
        Long.parseLong(args[i].substring(expectedCountArg.length())));
      continue;
    }
    // if no switch, assume column names
    sb.append(args[i]);
    sb.append(" ");
  }
  if (endTime < startTime) {
    printUsage("--endtime=" + endTime + " needs to be greater than --starttime=" + startTime);
    return null;
  }

  Job job = Job.getInstance(conf, conf.get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
  job.setJarByClass(RowCounter.class);
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  setScanFilter(scan, rowRangeList);
  if (sb.length() > 0) {
    for (String columnName : sb.toString().trim().split(" ")) {
      String family = StringUtils.substringBefore(columnName, ":");
      String qualifier = StringUtils.substringAfter(columnName, ":");

      if (StringUtils.isBlank(qualifier)) {
        scan.addFamily(Bytes.toBytes(family));
      }
      else {
        scan.addColumn(Bytes.toBytes(family), Bytes.toBytes(qualifier));
      }
    }
  }
  scan.setTimeRange(startTime, endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime);
  job.setOutputFormatClass(NullOutputFormat.class);
  TableMapReduceUtil.initTableMapperJob(tableName, scan,
    RowCounterMapper.class, ImmutableBytesWritable.class, Result.class, job);
  job.setNumReduceTasks(0);
  return job;
}
 

private void addColumnSetToScan(Scan scan, int[] colIds) {
  for (int colId : colIds) {
    scan.addColumn(FAMILY_BYTES,
        Bytes.toBytes(qualFromId(colId)));
  }
}