Java Code Examples for org.apache.hadoop.hbase.client.Scan.addFamily()

The following are Jave code examples for showing how to use addFamily() of the org.apache.hadoop.hbase.client.Scan class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: ditb   File: TestTableInputFormat.java   Source Code and License Vote up 7 votes
@Override
public void configure(JobConf job) {
  try {
    Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(job));
    TableName tableName = TableName.valueOf("exampleJobConfigurableTable");
    // mandatory
    initializeTable(connection, tableName);
    byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
      Bytes.toBytes("columnB") };
    //optional
    Scan scan = new Scan();
    for (byte[] family : inputColumns) {
      scan.addFamily(family);
    }
    Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
    scan.setFilter(exampleFilter);
    setScan(scan);
  } catch (IOException exception) {
    throw new RuntimeException("Failed to initialize.", exception);
  }
}
 
Example 2
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 6 votes
@Test (timeout=300000)
public void testMinWithInvalidRange2() {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(ROWS[6]);
  scan.setStopRow(ROWS[6]);
  final ColumnInterpreter<BigDecimal, BigDecimal, EmptyMsg, BigDecimalMsg, BigDecimalMsg> ci =
    new BigDecimalColumnInterpreter();
  BigDecimal min = null;
  try {
    min = aClient.min(TEST_TABLE, ci, scan);
  } catch (Throwable e) {
  }
  assertEquals(null, min);// control should go to the catch block
}
 
Example 3
Project: ditb   File: TestAggregateProtocol.java   Source Code and License Vote up 6 votes
@Test (timeout=300000)
public void testMinWithInvalidRange2() {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(ROWS[6]);
  scan.setStopRow(ROWS[6]);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Long min = null;
  try {
    min = aClient.min(TEST_TABLE, ci, scan);
  } catch (Throwable e) {
  }
  assertEquals(null, min);// control should go to the catch block
}
 
Example 4
Project: ditb   File: TestTableInputFormat.java   Source Code and License Vote up 6 votes
@Override
protected void initialize(JobContext job) throws IOException {
  Connection connection = ConnectionFactory.createConnection(HBaseConfiguration.create(
      job.getConfiguration()));
  TableName tableName = TableName.valueOf("exampleTable");
  // mandatory
  initializeTable(connection, tableName);
  byte[][] inputColumns = new byte [][] { Bytes.toBytes("columnA"),
    Bytes.toBytes("columnB") };
  //optional
  Scan scan = new Scan();
  for (byte[] family : inputColumns) {
    scan.addFamily(family);
  }
  Filter exampleFilter = new RowFilter(CompareOp.EQUAL, new RegexStringComparator("aa.*"));
  scan.setFilter(exampleFilter);
  setScan(scan);
}
 
Example 5
Project: ditb   File: TestTableInputFormatScanBase.java   Source Code and License Vote up 6 votes
/**
 * Tests a MR scan using data skew auto-balance
 *
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
public void testNumOfSplits(String ratio, int expectedNumOfSplits) throws IOException,
        InterruptedException,
        ClassNotFoundException {
  String jobName = "TestJobForNumOfSplits";
  LOG.info("Before map/reduce startup - job " + jobName);
  Configuration c = new Configuration(TEST_UTIL.getConfiguration());
  Scan scan = new Scan();
  scan.addFamily(INPUT_FAMILY);
  c.set("hbase.mapreduce.input.autobalance", "true");
  c.set("hbase.mapreduce.input.autobalance.maxskewratio", ratio);
  c.set(KEY_STARTROW, "");
  c.set(KEY_LASTROW, "");
  Job job = new Job(c, jobName);
  TableMapReduceUtil.initTableMapperJob(Bytes.toString(TABLE_NAME), scan, ScanMapper.class,
          ImmutableBytesWritable.class, ImmutableBytesWritable.class, job);
  TableInputFormat tif = new TableInputFormat();
  tif.setConf(job.getConfiguration());
  Assert.assertEquals(new String(TABLE_NAME), new String(table.getTableName()));
  List<InputSplit> splits = tif.getSplits(job);
  Assert.assertEquals(expectedNumOfSplits, splits.size());
}
 
Example 6
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 6 votes
@Test (timeout=300000)
public void testSumWithInvalidRange() {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(ROWS[6]);
  scan.setStopRow(ROWS[2]);
  final ColumnInterpreter<BigDecimal, BigDecimal, EmptyMsg, BigDecimalMsg, BigDecimalMsg> ci =
    new BigDecimalColumnInterpreter();
  BigDecimal sum = null;
  try {
    sum = aClient.sum(TEST_TABLE, ci, scan);
  } catch (Throwable e) {
  }
  assertEquals(null, sum);// control should go to the catch block
}
 
Example 7
Project: ditb   File: MDIndex.java   Source Code and License Vote up 6 votes
public Result getRowOrBefore(Table table, byte[] row, byte[] family) throws IOException {
  long start = System.currentTimeMillis();
  Scan scan = new Scan();
  scan.addFamily(family);
  scan.setReversed(true);
  scan.setStartRow(row);
  scan.setCacheBlocks(false);
  scan.setCaching(1);
  scan.setSmall(true);
  ResultScanner scanner = table.getScanner(scan);
  Result ret = scanner.next();
  scanner.close();
  prevRowTotalTime += System.currentTimeMillis() - start;
  prevRowTotalCount++;
  return ret;
}
 
Example 8
Project: ditb   File: TestAggregateProtocol.java   Source Code and License Vote up 6 votes
@Test (timeout=300000)
public void testStdWithInvalidRange() {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(ROWS[6]);
  scan.setStopRow(ROWS[1]);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  Double std = null;
  try {
    std = aClient.std(TEST_TABLE, ci, scan);
  } catch (Throwable e) {
  }
  assertEquals(null, std);// control should go to the catch block
}
 
Example 9
Project: ditb   File: TestBulkDeleteProtocol.java   Source Code and License Vote up 6 votes
public void testBulkDeleteFamily() throws Throwable {
  TableName tableName = TableName.valueOf("testBulkDeleteFamily");
  HTableDescriptor htd = new HTableDescriptor(tableName);
  htd.addFamily(new HColumnDescriptor(FAMILY1));
  htd.addFamily(new HColumnDescriptor(FAMILY2));
  TEST_UTIL.getHBaseAdmin().createTable(htd, Bytes.toBytes(0), Bytes.toBytes(120), 5);
  Table ht = new HTable(TEST_UTIL.getConfiguration(), tableName);
  List<Put> puts = new ArrayList<Put>(100);
  for (int j = 0; j < 100; j++) {
    Put put = new Put(Bytes.toBytes(j));
    put.add(FAMILY1, QUALIFIER1, "v1".getBytes());
    put.add(FAMILY2, QUALIFIER2, "v2".getBytes());
    puts.add(put);
  }
  ht.put(puts);
  Scan scan = new Scan();
  scan.addFamily(FAMILY1);
  // Delete the column family cf1
  long noOfRowsDeleted = invokeBulkDeleteProtocol(tableName, scan, 500, DeleteType.FAMILY, null);
  assertEquals(100, noOfRowsDeleted);
  int rows = 0;
  for (Result result : ht.getScanner(new Scan())) {
    assertTrue(result.getFamilyMap(FAMILY1).isEmpty());
    assertEquals(1, result.getColumnCells(FAMILY2, QUALIFIER2).size());
    rows++;
  }
  assertEquals(100, rows);
  ht.close();
}
 
Example 10
Project: ditb   File: TestHRegion.java   Source Code and License Vote up 6 votes
private InternalScanner buildScanner(String keyPrefix, String value, HRegion r)
    throws IOException {
  // Defaults FilterList.Operator.MUST_PASS_ALL.
  FilterList allFilters = new FilterList();
  allFilters.addFilter(new PrefixFilter(Bytes.toBytes(keyPrefix)));
  // Only return rows where this column value exists in the row.
  SingleColumnValueFilter filter = new SingleColumnValueFilter(Bytes.toBytes("trans-tags"),
      Bytes.toBytes("qual2"), CompareOp.EQUAL, Bytes.toBytes(value));
  filter.setFilterIfMissing(true);
  allFilters.addFilter(filter);
  Scan scan = new Scan();
  scan.addFamily(Bytes.toBytes("trans-blob"));
  scan.addFamily(Bytes.toBytes("trans-type"));
  scan.addFamily(Bytes.toBytes("trans-date"));
  scan.addFamily(Bytes.toBytes("trans-tags"));
  scan.addFamily(Bytes.toBytes("trans-group"));
  scan.setFilter(allFilters);
  return r.getScanner(scan);
}
 
Example 11
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 5 votes
@Test (timeout=300000)
public void testAvgWithValidRangeWithNoCQ() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  final ColumnInterpreter<BigDecimal, BigDecimal, EmptyMsg, BigDecimalMsg, BigDecimalMsg> ci =
    new BigDecimalColumnInterpreter();
  double avg = aClient.avg(TEST_TABLE, ci, scan);
  assertEquals(10.45, avg, 0.01);
}
 
Example 12
Project: ditb   File: TestSeekBeforeWithReverseScan.java   Source Code and License Vote up 5 votes
@Test
public void testReverseScanWithPadding() throws Exception {
  byte[] terminator = new byte[] { -1 };
  byte[] row1 = Bytes.add(invert(Bytes.toBytes("a")), terminator);
  byte[] row2 = Bytes.add(invert(Bytes.toBytes("ab")), terminator);
  byte[] row3 = Bytes.add(invert(Bytes.toBytes("b")), terminator);

  Put put1 = new Put(row1);
  put1.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);
  Put put2 = new Put(row2);
  put2.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);
  Put put3 = new Put(row3);
  put3.addColumn(cfName, cqName, HConstants.EMPTY_BYTE_ARRAY);

  region.put(put1);
  region.put(put2);
  region.put(put3);
  region.flush(true);
  Scan scan = new Scan();
  scan.setCacheBlocks(false);
  scan.setReversed(true);
  scan.setFilter(new FirstKeyOnlyFilter());
  scan.addFamily(cfName);
  RegionScanner scanner = region.getScanner(scan);
  List<Cell> res = new ArrayList<Cell>();
  int count = 1;
  while (scanner.next(res)) {
    count++;
  }
  assertEquals(3, count);
}
 
Example 13
Project: ditb   File: TestAggregateProtocol.java   Source Code and License Vote up 5 votes
@Test (timeout=300000)
public void testMinWithValidRangeWithNoCQ() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(HConstants.EMPTY_START_ROW);
  scan.setStopRow(HConstants.EMPTY_END_ROW);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  long min = aClient.min(TEST_TABLE, ci,
      scan);
  assertEquals(0, min);
}
 
Example 14
Project: ditb   File: TestSCVFWithMiniCluster.java   Source Code and License Vote up 5 votes
/**
 * Test the filter by adding all columns of family A in the scan. (OK)
 */
@Test
public void scanWithAllQualifiersOfFamiliyA() throws IOException {
  /* Given */
  Scan scan = new Scan();
  scan.addFamily(FAMILY_A);
  scan.setFilter(scanFilter);

  verify(scan);
}
 
Example 15
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 5 votes
@Test (timeout=300000)
public void testSumWithValidRangeWithNoCQ() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  final ColumnInterpreter<BigDecimal, BigDecimal, EmptyMsg, BigDecimalMsg, BigDecimalMsg> ci =
    new BigDecimalColumnInterpreter();
  BigDecimal sum = aClient.sum(TEST_TABLE, ci, scan);
  assertEquals(new BigDecimal("209.00"), sum); // 190 + 19
}
 
Example 16
Project: ditb   File: TestBigDecimalColumnInterpreter.java   Source Code and License Vote up 5 votes
@Test (timeout=300000)
public void testMinWithValidRange2WithNoCQ() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  scan.setStartRow(ROWS[6]);
  scan.setStopRow(ROWS[7]);
  final ColumnInterpreter<BigDecimal, BigDecimal, EmptyMsg, BigDecimalMsg, BigDecimalMsg> ci =
    new BigDecimalColumnInterpreter();
  BigDecimal min = aClient.min(TEST_TABLE, ci, scan);
  assertEquals(new BigDecimal("0.60"), min);
}
 
Example 17
Project: ditb   File: ThriftServerRunner.java   Source Code and License Vote up 5 votes
@Override
public int scannerOpenTs(ByteBuffer tableName, ByteBuffer startRow,
    List<ByteBuffer> columns, long timestamp,
    Map<ByteBuffer, ByteBuffer> attributes) throws IOError, TException {

  Table table = null;
  try {
    table = getTable(tableName);
    Scan scan = new Scan(getBytes(startRow));
    addAttributes(scan, attributes);
    scan.setTimeRange(0, timestamp);
    if (columns != null && columns.size() != 0) {
      for (ByteBuffer column : columns) {
        byte [][] famQf = KeyValue.parseColumn(getBytes(column));
        if(famQf.length == 1) {
          scan.addFamily(famQf[0]);
        } else {
          scan.addColumn(famQf[0], famQf[1]);
        }
      }
    }
    return addScanner(table.getScanner(scan), false);
  } catch (IOException e) {
    LOG.warn(e.getMessage(), e);
    throw new IOError(Throwables.getStackTraceAsString(e));
  } finally{
    closeTable(table);
  }
}
 
Example 18
Project: ditb   File: TestAggregateProtocol.java   Source Code and License Vote up 5 votes
@Test (timeout=300000)
public void testMaxWithValidRangeWithNoCQ() throws Throwable {
  AggregationClient aClient = new AggregationClient(conf);
  Scan scan = new Scan();
  scan.addFamily(TEST_FAMILY);
  final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
      new LongColumnInterpreter();
  long maximum = aClient.max(TEST_TABLE, ci, scan);
  assertEquals(190, maximum);
}
 
Example 19
Project: ditb   File: TestMultithreadedTableMapper.java   Source Code and License Vote up 4 votes
/**
 * Looks at every value of the mapreduce output and verifies that indeed
 * the values have been reversed.
 *
 * @param table Table to scan.
 * @throws IOException
 * @throws NullPointerException if we failed to find a cell value
 */
private void verifyAttempt(final Table table)
    throws IOException, NullPointerException {
  Scan scan = new Scan();
  scan.addFamily(INPUT_FAMILY);
  scan.addFamily(OUTPUT_FAMILY);
  ResultScanner scanner = table.getScanner(scan);
  try {
    Iterator<Result> itr = scanner.iterator();
    assertTrue(itr.hasNext());
    while(itr.hasNext()) {
      Result r = itr.next();
      if (LOG.isDebugEnabled()) {
        if (r.size() > 2 ) {
          throw new IOException("Too many results, expected 2 got " +
              r.size());
        }
      }
      byte[] firstValue = null;
      byte[] secondValue = null;
      int count = 0;
      for(Cell kv : r.listCells()) {
        if (count == 0) {
          firstValue = CellUtil.cloneValue(kv);
        }else if (count == 1) {
          secondValue = CellUtil.cloneValue(kv);
        }else if (count == 2) {
          break;
        }
        count++;
      }
      String first = "";
      if (firstValue == null) {
        throw new NullPointerException(Bytes.toString(r.getRow()) +
            ": first value is null");
      }
      first = Bytes.toString(firstValue);
      String second = "";
      if (secondValue == null) {
        throw new NullPointerException(Bytes.toString(r.getRow()) +
            ": second value is null");
      }
      byte[] secondReversed = new byte[secondValue.length];
      for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
        secondReversed[i] = secondValue[j];
      }
      second = Bytes.toString(secondReversed);
      if (first.compareTo(second) != 0) {
        if (LOG.isDebugEnabled()) {
          LOG.debug("second key is not the reverse of first. row=" +
              Bytes.toStringBinary(r.getRow()) + ", first value=" + first +
              ", second value=" + second);
        }
        fail();
      }
    }
  } finally {
    scanner.close();
  }
}
 
Example 20
Project: ditb   File: IntegrationTestLazyCfLoading.java   Source Code and License Vote up 4 votes
@Test
public void testReadersAndWriters() throws Exception {
  Configuration conf = util.getConfiguration();
  String timeoutKey = String.format(TIMEOUT_KEY, this.getClass().getSimpleName());
  long maxRuntime = conf.getLong(timeoutKey, DEFAULT_TIMEOUT_MINUTES);
  long serverCount = util.getHBaseClusterInterface().getClusterStatus().getServersSize();
  long keysToWrite = serverCount * KEYS_TO_WRITE_PER_SERVER;
  Table table = new HTable(conf, TABLE_NAME);

  // Create multi-threaded writer and start it. We write multiple columns/CFs and verify
  // their integrity, therefore multi-put is necessary.
  MultiThreadedWriter writer =
    new MultiThreadedWriter(dataGen, conf, TABLE_NAME);
  writer.setMultiPut(true);

  LOG.info("Starting writer; the number of keys to write is " + keysToWrite);
  // TODO : Need to see if tag support has to be given here in the integration test suite
  writer.start(1, keysToWrite, WRITER_THREADS);

  // Now, do scans.
  long now = EnvironmentEdgeManager.currentTime();
  long timeLimit = now + (maxRuntime * 60000);
  boolean isWriterDone = false;
  while (now < timeLimit && !isWriterDone) {
    LOG.info("Starting the scan; wrote approximately "
      + dataGen.getTotalNumberOfKeys() + " keys");
    isWriterDone = writer.isDone();
    if (isWriterDone) {
      LOG.info("Scanning full result, writer is done");
    }
    Scan scan = new Scan();
    for (byte[] cf : dataGen.getColumnFamilies()) {
      scan.addFamily(cf);
    }
    scan.setFilter(dataGen.getScanFilter());
    scan.setLoadColumnFamiliesOnDemand(true);
    // The number of keys we can expect from scan - lower bound (before scan).
    // Not a strict lower bound - writer knows nothing about filters, so we report
    // this from generator. Writer might have generated the value but not put it yet.
    long onesGennedBeforeScan = dataGen.getExpectedNumberOfKeys();
    long startTs = EnvironmentEdgeManager.currentTime();
    ResultScanner results = table.getScanner(scan);
    long resultCount = 0;
    Result result = null;
    // Verify and count the results.
    while ((result = results.next()) != null) {
      boolean isOk = writer.verifyResultAgainstDataGenerator(result, true, true);
      Assert.assertTrue("Failed to verify [" + Bytes.toString(result.getRow())+ "]", isOk);
      ++resultCount;
    }
    long timeTaken = EnvironmentEdgeManager.currentTime() - startTs;
    // Verify the result count.
    long onesGennedAfterScan = dataGen.getExpectedNumberOfKeys();
    Assert.assertTrue("Read " + resultCount + " keys when at most " + onesGennedAfterScan
      + " were generated ", onesGennedAfterScan >= resultCount);
    if (isWriterDone) {
      Assert.assertTrue("Read " + resultCount + " keys; the writer is done and "
        + onesGennedAfterScan + " keys were generated", onesGennedAfterScan == resultCount);
    } else if (onesGennedBeforeScan * 0.9 > resultCount) {
      LOG.warn("Read way too few keys (" + resultCount + "/" + onesGennedBeforeScan
        + ") - there might be a problem, or the writer might just be slow");
    }
    LOG.info("Scan took " + timeTaken + "ms");
    if (!isWriterDone) {
      Thread.sleep(WAIT_BETWEEN_SCANS_MS);
      now = EnvironmentEdgeManager.currentTime();
    }
  }
  Assert.assertEquals("There are write failures", 0, writer.getNumWriteFailures());
  Assert.assertTrue("Writer is not done", isWriterDone);
  // Assert.fail("Boom!");
}