Java Code Examples for org.apache.hadoop.hbase.client.Scan#readVersions()

The following examples show how to use org.apache.hadoop.hbase.client.Scan#readVersions() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   File: TestKeepDeletes.java    License: Apache License 2.0 6 votes vote down vote up
private int countDeleteMarkers(HRegion region) throws IOException {
  Scan s = new Scan();
  s.setRaw(true);
  // use max versions from the store(s)
  s.readVersions(region.getStores().iterator().next().getScanInfo().getMaxVersions());
  InternalScanner scan = region.getScanner(s);
  List<Cell> kvs = new ArrayList<>();
  int res = 0;
  boolean hasMore;
  do {
    hasMore = scan.next(kvs);
    for (Cell kv : kvs) {
      if(CellUtil.isDelete(kv)) {
        res++;
      }
    }
    kvs.clear();
  } while (hasMore);
  scan.close();
  return res;
}
 
Example 2
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 3
Source Project: atlas   File: HBaseAdmin2_0.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Delete all rows from the given table. This method is intended only for development and testing use.
 * @param tableString
 * @param timestamp
 * @throws IOException
 */
@Override
public void clearTable(String tableString, long timestamp) throws IOException
{
    TableName tableName = TableName.valueOf(tableString);

    if (!adm.tableExists(tableName)) {
        log.debug("Attempted to clear table {} before it exists (noop)", tableString);
        return;
    }

    // Unfortunately, linear scanning and deleting rows is faster in HBase when running integration tests than
    // disabling and deleting/truncating tables.
    final Scan scan = new Scan();
    scan.setCacheBlocks(false);
    scan.setCaching(2000);
    scan.setTimeRange(0, Long.MAX_VALUE);
    scan.readVersions(1);

    try (final Table table = adm.getConnection().getTable(tableName);
         final ResultScanner scanner = table.getScanner(scan)) {
        final Iterator<Result> iterator = scanner.iterator();
        final int batchSize = 1000;
        final List<Delete> deleteList = new ArrayList<>();
        while (iterator.hasNext()) {
            deleteList.add(new Delete(iterator.next().getRow(), timestamp));
            if (!iterator.hasNext() || deleteList.size() == batchSize) {
                table.delete(deleteList);
                deleteList.clear();
            }
        }
    }
}
 
Example 4
Source Project: hbase   File: BackupSystemTable.java    License: Apache License 2.0 5 votes vote down vote up
static Scan createScanForOrigBulkLoadedFiles(TableName table) {
  Scan scan = new Scan();
  byte[] startRow = rowkey(BULK_LOAD_PREFIX, table.toString(), BLK_LD_DELIM);
  byte[] stopRow = Arrays.copyOf(startRow, startRow.length);
  stopRow[stopRow.length - 1] = (byte) (stopRow[stopRow.length - 1] + 1);
  scan.withStartRow(startRow);
  scan.withStopRow(stopRow);
  scan.addFamily(BackupSystemTable.META_FAMILY);
  scan.readVersions(1);
  return scan;
}
 
Example 5
Source Project: hbase   File: TestCoprocessorScanPolicy.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Scan scan)
    throws IOException {
  Region region = c.getEnvironment().getRegion();
  TableName tableName = region.getTableDescriptor().getTableName();
  Long ttl = this.ttls.get(tableName);
  if (ttl != null) {
    scan.setTimeRange(EnvironmentEdgeManager.currentTime() - ttl, scan.getTimeRange().getMax());
  }
  Integer version = this.versions.get(tableName);
  if (version != null) {
    scan.readVersions(version);
  }
}
 
Example 6
Source Project: hbase   File: MobRefReporter.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Main method for the tool.
 * @return 0 if success, 1 for bad args. 2 if job aborted with an exception,
 *   3 if mr job was unsuccessful
 */
public int run(String[] args) throws IOException, InterruptedException {
  // TODO make family and table optional
  if (args.length != 3) {
    printUsage();
    return 1;
  }
  final String output = args[0];
  final String tableName = args[1];
  final String familyName = args[2];
  final long reportStartTime = EnvironmentEdgeManager.currentTime();
  Configuration conf = getConf();
  try {
    FileSystem fs = FileSystem.get(conf);
    // check whether the current user is the same one with the owner of hbase root
    String currentUserName = UserGroupInformation.getCurrentUser().getShortUserName();
    FileStatus[] hbaseRootFileStat = fs.listStatus(new Path(conf.get(HConstants.HBASE_DIR)));
    if (hbaseRootFileStat.length > 0) {
      String owner = hbaseRootFileStat[0].getOwner();
      if (!owner.equals(currentUserName)) {
        String errorMsg = "The current user[" + currentUserName
            + "] does not have hbase root credentials."
            + " If this job fails due to an inability to read HBase's internal directories, "
            + "you will need to rerun as a user with sufficient permissions. The HBase superuser "
            + "is a safe choice.";
        LOG.warn(errorMsg);
      }
    } else {
      LOG.error("The passed configs point to an HBase dir does not exist: {}",
          conf.get(HConstants.HBASE_DIR));
      throw new IOException("The target HBase does not exist");
    }

    byte[] family;
    int maxVersions;
    TableName tn = TableName.valueOf(tableName);
    try (Connection connection = ConnectionFactory.createConnection(conf);
         Admin admin = connection.getAdmin()) {
      TableDescriptor htd = admin.getDescriptor(tn);
      ColumnFamilyDescriptor hcd = htd.getColumnFamily(Bytes.toBytes(familyName));
      if (hcd == null || !hcd.isMobEnabled()) {
        throw new IOException("Column family " + familyName + " is not a MOB column family");
      }
      family = hcd.getName();
      maxVersions = hcd.getMaxVersions();
    }


    String id = getClass().getSimpleName() + UUID.randomUUID().toString().replace("-", "");
    Job job = null;
    Scan scan = new Scan();
    scan.addFamily(family);
    // Do not retrieve the mob data when scanning
    scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
    scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE));
    // If a scanner caching value isn't set, pick a smaller default since we know we're doing
    // a full table scan and don't want to impact other clients badly.
    scan.setCaching(conf.getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 10000));
    scan.setCacheBlocks(false);
    scan.readVersions(maxVersions);
    conf.set(REPORT_JOB_ID, id);

    job = Job.getInstance(conf);
    job.setJarByClass(getClass());
    TableMapReduceUtil.initTableMapperJob(tn, scan,
        MobRefMapper.class, Text.class, ImmutableBytesWritable.class, job);

    job.setReducerClass(MobRefReducer.class);
    job.setOutputFormatClass(TextOutputFormat.class);
    TextOutputFormat.setOutputPath(job, new Path(output));

    job.setJobName(getClass().getSimpleName() + "-" + tn + "-" + familyName);
    // for use in the reducer. easier than re-parsing it out of the scan string.
    job.getConfiguration().set(TableInputFormat.SCAN_COLUMN_FAMILY, familyName);

    // Use when we start this job as the base point for file "recency".
    job.getConfiguration().setLong(REPORT_START_DATETIME, reportStartTime);

    if (job.waitForCompletion(true)) {
      LOG.info("Finished creating report for '{}', family='{}'", tn, familyName);
    } else {
      System.err.println("Job was not successful");
      return 3;
    }
    return 0;

  } catch (ClassNotFoundException | RuntimeException | IOException | InterruptedException e) {
    System.err.println("Job aborted due to exception " + e);
    return 2; // job failed
  }
}
 
Example 7
Source Project: hbase   File: TestScannerWithBulkload.java    License: Apache License 2.0 4 votes vote down vote up
private Scan createScan() {
  Scan scan = new Scan();
  scan.readVersions(3);
  return scan;
}
 
Example 8
private void testFlushFile(TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor)
    throws Exception {
  Table table = null;
  try {
    table = TEST_UTIL.createTable(tableDescriptor, null);

    //put data
    Put put0 = new Put(row1);
    put0.addColumn(family, qf1, 1, value1);
    table.put(put0);

    //put more data
    Put put1 = new Put(row2);
    put1.addColumn(family, qf2, 1, value2);
    table.put(put1);

    //flush
    TEST_UTIL.flush(tableDescriptor.getTableName());

    //Scan
    Scan scan = new Scan();
    scan.addColumn(family, qf1);
    scan.readVersions(4);
    ResultScanner scanner = table.getScanner(scan);

    //Compare
    int size = 0;
    for (Result result : scanner) {
      size++;
      List<Cell> cells = result.getColumnCells(family, qf1);
      // Verify the cell size
      Assert.assertEquals(1, cells.size());
      // Verify the value
      Assert.assertArrayEquals(value1, CellUtil.cloneValue(cells.get(0)));
    }
    scanner.close();
    Assert.assertEquals(1, size);
  } finally {
    table.close();
  }
}
 
Example 9
@Test
public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersions() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(
              "(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + SECRET + "&" + TOPSECRET + ")"));
          d.addColumns(fam, qual, 125L);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getAdmin().flush(tableName);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(125L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}
 
Example 10
@Test
public void testVisibilityLabelsWithDeleteColumnsWithMultipleVersionsNoTimestamp()
    throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d1 = new Delete(row1);
          d1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
          d1.addColumns(fam, qual);

          table.delete(d1);

          Delete d2 = new Delete(row1);
          d2.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
          d2.addColumns(fam, qual);
          table.delete(d2);

          Delete d3 = new Delete(row1);
          d3.setCellVisibility(new CellVisibility(
              "(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + SECRET + "&" + TOPSECRET + ")"));
          d3.addColumns(fam, qual);
          table.delete(d3);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertEquals(1, next.length);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}
 
Example 11
Source Project: hbase   File: ScannerResultGenerator.java    License: Apache License 2.0 4 votes vote down vote up
public ScannerResultGenerator(final String tableName, final RowSpec rowspec,
  final Filter filter, final int caching ,final boolean cacheBlocks, int limit) throws IOException {
  Table table = RESTServlet.getInstance().getTable(tableName);
  try {
    Scan scan;
    if (rowspec.hasEndRow()) {
      scan = new Scan().withStartRow(rowspec.getStartRow()).withStopRow(rowspec.getEndRow());
    } else {
      scan = new Scan().withStartRow(rowspec.getStartRow());
    }
    if (rowspec.hasColumns()) {
      byte[][] columns = rowspec.getColumns();
      for (byte[] column: columns) {
        byte[][] split = CellUtil.parseColumn(column);
        if (split.length == 1) {
          scan.addFamily(split[0]);
        } else if (split.length == 2) {
          scan.addColumn(split[0], split[1]);
        } else {
          throw new IllegalArgumentException("Invalid familyAndQualifier provided.");
        }
      }
    }
    scan.setTimeRange(rowspec.getStartTime(), rowspec.getEndTime());
    scan.readVersions(rowspec.getMaxVersions());
    if (filter != null) {
      scan.setFilter(filter);
    }
    if (caching > 0 ) {
      scan.setCaching(caching);
    }
    if (limit > 0) {
      scan.setLimit(limit);
    }
    scan.setCacheBlocks(cacheBlocks);
    if (rowspec.hasLabels()) {
      scan.setAuthorizations(new Authorizations(rowspec.getLabels()));
    }
    scanner = table.getScanner(scan);
    cached = null;
    id = Long.toString(System.currentTimeMillis()) +
           Integer.toHexString(scanner.hashCode());
  } finally {
    table.close();
  }
}
 
Example 12
@Test
public void testVisibilityLabelsWithDeleteFamilyNoMatchingVisExpWithMultipleVersionsNoTimestamp()
    throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        Delete d1 = new Delete(row1);
        d1.setCellVisibility(new CellVisibility(CONFIDENTIAL));
        d1.addFamily(fam);

        Delete d2 = new Delete(row1);
        d2.setCellVisibility(new CellVisibility(SECRET));
        d2.addFamily(fam);

        Delete d3 = new Delete(row1);
        d3.setCellVisibility(new CellVisibility(
            "(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + SECRET + "&" + TOPSECRET + ")"));
        d3.addFamily(fam);

        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          table.delete(createList(d1, d2, d3));
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
    scanner.close();
  }
}
 
Example 13
@Test
public void testDeleteFamilyAndDeleteColumnsWithAndWithoutVisibilityExp() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        Delete d1 = new Delete(row1);
        d1.addFamily(fam);

        Delete d2 = new Delete(row1);
        d2.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
        d2.addColumns(fam, qual);
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          table.delete(createList(d1, d2));
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(124L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
    scanner.close();
  }
}
 
Example 14
@Test
public void testDeleteFamilySpecificTimeStampWithMulipleVersions() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(
              "(" + PRIVATE + "&" + CONFIDENTIAL + ")|(" + SECRET + "&" + TOPSECRET + ")"));
          d.addFamily(fam, 126L);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getAdmin().flush(tableName);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(6);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(125L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}
 
Example 15
@Test
public void testDeleteColumnWithLatestTimeStampUsingMultipleVersions() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
          d.addColumn(fam, qual);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getAdmin().flush(tableName);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(124L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}
 
Example 16
@Test
public void testDeleteColumnWithLatestTimeStampUsingMultipleVersionsAfterCompaction()
    throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPuts(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
          d.addColumn(fam, qual);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    TEST_UTIL.getAdmin().flush(tableName);
    Put put = new Put(Bytes.toBytes("row3"));
    put.addColumn(fam, qual, 127L, value);
    put.setCellVisibility(new CellVisibility(CONFIDENTIAL + "&" + PRIVATE));
    table.put(put);
    TEST_UTIL.getAdmin().flush(tableName);
    TEST_UTIL.getAdmin().majorCompact(tableName);
    // Sleep to ensure compaction happens. Need to do it in a better way
    Thread.sleep(5000);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 3);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(124L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}
 
Example 17
Source Project: hbase   File: TestWideScanner.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testWideScanBatching() throws IOException {
  final int batch = 256;
  int inserted = addWideContent(REGION);
  List<Cell> results = new ArrayList<>();
  Scan scan = new Scan();
  scan.addFamily(A);
  scan.addFamily(B);
  scan.addFamily(C);
  scan.readVersions(100);
  scan.setBatch(batch);
  try (InternalScanner s = REGION.getScanner(scan)) {
    int total = 0;
    int i = 0;
    boolean more;
    do {
      more = s.next(results);
      i++;
      LOG.info("iteration #" + i + ", results.size=" + results.size());

      // assert that the result set is no larger
      assertTrue(results.size() <= batch);

      total += results.size();

      if (results.size() > 0) {
        // assert that all results are from the same row
        byte[] row = CellUtil.cloneRow(results.get(0));
        for (Cell kv : results) {
          assertTrue(Bytes.equals(row, CellUtil.cloneRow(kv)));
        }
      }

      results.clear();

      // trigger ChangedReadersObservers
      Iterator<KeyValueScanner> scanners =
        ((HRegion.RegionScannerImpl) s).storeHeap.getHeap().iterator();
      while (scanners.hasNext()) {
        StoreScanner ss = (StoreScanner) scanners.next();
        ss.updateReaders(Collections.emptyList(), Collections.emptyList());
      }
    } while (more);

    // assert that the scanner returned all values
    LOG.info("inserted " + inserted + ", scanned " + total);
    assertEquals(total, inserted);
  }
}
 
Example 18
@Test
public void testDeleteColumnswithMultipleColumnsWithMultipleVersions() throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPutsWithDiffCols(tableName)) {
    TEST_UTIL.getAdmin().flush(tableName);
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        Delete d = new Delete(row1);
        d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
        d.addColumns(fam, qual, 125L);
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);

    TEST_UTIL.getAdmin().flush(tableName);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 1);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(124L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
      current.getQualifierLength(), qual1, 0, qual1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    assertTrue(Bytes.equals(current.getQualifierArray(), current.getQualifierOffset(),
      current.getQualifierLength(), qual2, 0, qual2.length));
  }
}
 
Example 19
Source Project: hbase   File: IntegrationTestBulkLoad.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * After adding data to the table start a mr job to
 * @throws IOException
 * @throws ClassNotFoundException
 * @throws InterruptedException
 */
private void runCheck() throws IOException, ClassNotFoundException, InterruptedException {
  LOG.info("Running check");
  Configuration conf = getConf();
  String jobName = getTablename() + "_check" + EnvironmentEdgeManager.currentTime();
  Path p = util.getDataTestDirOnTestFS(jobName);

  Job job = new Job(conf);
  job.setJarByClass(getClass());
  job.setJobName(jobName);

  job.setPartitionerClass(NaturalKeyPartitioner.class);
  job.setGroupingComparatorClass(NaturalKeyGroupingComparator.class);
  job.setSortComparatorClass(CompositeKeyComparator.class);

  Scan scan = new Scan();
  scan.addFamily(CHAIN_FAM);
  scan.addFamily(SORT_FAM);
  scan.readVersions(1);
  scan.setCacheBlocks(false);
  scan.setBatch(1000);

  int replicaCount = conf.getInt(NUM_REPLICA_COUNT_KEY, NUM_REPLICA_COUNT_DEFAULT);
  if (replicaCount != NUM_REPLICA_COUNT_DEFAULT) {
    scan.setConsistency(Consistency.TIMELINE);
  }

  TableMapReduceUtil.initTableMapperJob(
      getTablename().getName(),
      scan,
      LinkedListCheckingMapper.class,
      LinkKey.class,
      LinkChain.class,
      job
  );

  job.setReducerClass(LinkedListCheckingReducer.class);
  job.setOutputKeyClass(NullWritable.class);
  job.setOutputValueClass(NullWritable.class);

  FileOutputFormat.setOutputPath(job, p);

  assertEquals(true, job.waitForCompletion(true));

  // Delete the files.
  util.getTestFileSystem().delete(p, true);
}
 
Example 20
@Test
public void testDeleteFamilyLatestTimeStampWithMulipleVersionsWithoutCellVisibilityInPuts()
    throws Exception {
  setAuths();
  final TableName tableName = TableName.valueOf(testName.getMethodName());
  try (Table table = doPutsWithoutVisibility(tableName)) {
    PrivilegedExceptionAction<Void> actiona = new PrivilegedExceptionAction<Void>() {
      @Override
      public Void run() throws Exception {
        try (Connection connection = ConnectionFactory.createConnection(conf);
          Table table = connection.getTable(tableName)) {
          Delete d = new Delete(row1);
          d.setCellVisibility(new CellVisibility(SECRET + "&" + TOPSECRET));
          d.addFamily(fam);
          table.delete(d);
        } catch (Throwable t) {
          throw new IOException(t);
        }
        return null;
      }
    };
    SUPERUSER.runAs(actiona);
    TEST_UTIL.getAdmin().flush(tableName);
    Scan s = new Scan();
    s.readVersions(5);
    s.setAuthorizations(new Authorizations(SECRET, PRIVATE, CONFIDENTIAL, TOPSECRET));
    ResultScanner scanner = table.getScanner(s);
    Result[] next = scanner.next(3);
    assertTrue(next.length == 2);
    CellScanner cellScanner = next[0].cellScanner();
    cellScanner.advance();
    Cell current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(127L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(126L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(125L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(124L, current.getTimestamp());
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row1, 0, row1.length));
    assertEquals(123L, current.getTimestamp());
    cellScanner = next[1].cellScanner();
    cellScanner.advance();
    current = cellScanner.current();
    assertTrue(Bytes.equals(current.getRowArray(), current.getRowOffset(), current.getRowLength(),
      row2, 0, row2.length));
  }
}