Java Code Examples for org.apache.hadoop.hbase.KeepDeletedCells

The following examples show how to use org.apache.hadoop.hbase.KeepDeletedCells. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
protected final MatchCode tryDropDelete(Cell cell) {
  long timestamp = cell.getTimestamp();
  // If it is not the time to drop the delete marker, just return
  if (timeToPurgeDeletes > 0 && now - timestamp <= timeToPurgeDeletes) {
    return MatchCode.INCLUDE;
  }
  if (keepDeletedCells == KeepDeletedCells.TRUE
      || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= oldestUnexpiredTS)) {
    // If keepDeletedCell is true, or the delete marker is not expired yet, we should include it
    // in version counting to see if we can drop it. The only exception is that, we can make
    // sure that no put is older than this delete marker. And under this situation, all later
    // cells of this column(must be delete markers) can be skipped.
    if (timestamp < earliestPutTs) {
      return columns.getNextRowOrNextColumn(cell);
    } else {
      return null;
    }
  } else {
    return MatchCode.SKIP;
  }
}
 
Example 2
Source Project: hbase   Source File: ScanInfo.java    License: Apache License 2.0 6 votes vote down vote up
private ScanInfo(byte[] family, int minVersions, int maxVersions, long ttl,
    KeepDeletedCells keepDeletedCells, long timeToPurgeDeletes, CellComparator comparator,
    long tableMaxRowSize, boolean usePread, long cellsPerTimeoutCheck,
    boolean parallelSeekEnabled, long preadMaxBytes, boolean newVersionBehavior) {
  this.family = family;
  this.minVersions = minVersions;
  this.maxVersions = maxVersions;
  this.ttl = ttl;
  this.keepDeletedCells = keepDeletedCells;
  this.timeToPurgeDeletes = timeToPurgeDeletes;
  this.comparator = comparator;
  this.tableMaxRowSize = tableMaxRowSize;
  this.usePread = usePread;
  this.cellsPerTimeoutCheck = cellsPerTimeoutCheck;
  this.parallelSeekEnabled = parallelSeekEnabled;
  this.preadMaxBytes = preadMaxBytes;
  this.newVersionBehavior = newVersionBehavior;
}
 
Example 3
Source Project: hbase   Source File: TestKeepDeletes.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * The ExplicitColumnTracker does not support "raw" scanning.
 */
@Test
public void testRawScanWithColumns() throws Exception {
  HTableDescriptor htd = hbu.createTableDescriptor(TableName.valueOf(name.getMethodName()), 0, 3,
      HConstants.FOREVER, KeepDeletedCells.TRUE);
  Region region = hbu.createLocalHRegion(htd, null, null);

  Scan s = new Scan();
  s.setRaw(true);
  s.readAllVersions();
  s.addColumn(c0, c0);

  try {
    region.getScanner(s);
    fail("raw scanner with columns should have failed");
  } catch (org.apache.hadoop.hbase.DoNotRetryIOException dnre) {
    // ok!
  }

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 
Example 4
Source Project: hbase   Source File: TestCompactionScanQueryMatcher.java    License: Apache License 2.0 6 votes vote down vote up
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example 5
Source Project: hbase   Source File: TestMinVersions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
  int ttl = 4;
  ColumnFamilyDescriptor cfd =
    ColumnFamilyDescriptorBuilder.newBuilder(c0)
      .setVersionsWithTimeToLive(ttl, 2).build();
  verifyVersionedCellKeyValues(ttl, cfd);

  cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0)
    .setMinVersions(2)
    .setMaxVersions(Integer.MAX_VALUE)
    .setTimeToLive(ttl)
    .setKeepDeletedCells(KeepDeletedCells.TTL)
    .build();
  verifyVersionedCellKeyValues(ttl, cfd);
}
 
Example 6
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 7
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadVersionWithRawAndFilter() throws IOException {
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
          , CellComparator.getInstance(), false);
  KeyValue [] kvs = new KeyValue[] {
          create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[]{
      new KeyValueScanFixture(CellComparator.getInstance(), kvs)
    });

  BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
  Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
  scanSpec.setFilter(filter);
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(2, results.size());
  }
}
 
Example 8
Source Project: phoenix   Source File: PropertiesInSyncIT.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Helper method to verify HBase column family properties
 * @param tableName Physical HBase table whose properties are to be verified
 * @param conn Phoenix connection
 * @param propModified true if we have altered any of the properties to be kept in sync, false otherwise
 * @param ignoreTTL We cannot modfiy a table level property when adding a column, so in those cases,
 *                 ignore the check for TTL modification
 * @throws Exception
 */
private void verifyHBaseColumnFamilyProperties(String tableName, Connection conn, boolean propModified,
        boolean ignoreTTL) throws Exception {
    final int expectedTTL = propModified ? MODIFIED_TTL_VALUE:INITIAL_TTL_VALUE;
    final KeepDeletedCells expectedKeepDeletedCells = propModified ? MODIFIED_KEEP_DELETED_CELLS_VALUE: INITIAL_KEEP_DELETED_CELLS_VALUE;
    final int expectedReplicationScope = propModified ? MODIFIED_REPLICATION_SCOPE_VALUE:INITIAL_REPLICATION_SCOPE_VALUE;

    try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
        // Note that this includes the local index column family as well
        ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(tableName))
                .getColumnFamilies();
        for (ColumnFamilyDescriptor cfd: columnFamilies) {
            if (!ignoreTTL) {
                assertEquals("Mismatch in TTL", expectedTTL, cfd.getTimeToLive());
            }
            assertEquals("Mismatch in KEEP_DELETED_CELLS", expectedKeepDeletedCells, cfd.getKeepDeletedCells());
            assertEquals("Mismatch in REPLICATION_SCOPE", expectedReplicationScope, cfd.getScope());
        }
    }
}
 
Example 9
Source Project: Halyard   Source File: HalyardTableUtils.java    License: Apache License 2.0 5 votes vote down vote up
private static HColumnDescriptor createColumnFamily() {
    return new HColumnDescriptor(CF_NAME)
            .setMaxVersions(1)
            .setBlockCacheEnabled(true)
            .setBloomFilterType(BloomType.ROW)
            .setCompressionType(DEFAULT_COMPRESSION_ALGORITHM)
            .setDataBlockEncoding(DEFAULT_DATABLOCK_ENCODING)
            .setCacheBloomsOnWrite(true)
            .setCacheDataOnWrite(true)
            .setCacheIndexesOnWrite(true)
            .setKeepDeletedCells(KeepDeletedCells.FALSE)
            .setValue(HTableDescriptor.MAX_FILESIZE, REGION_MAX_FILESIZE)
            .setValue(HTableDescriptor.SPLIT_POLICY, REGION_SPLIT_POLICY);
}
 
Example 10
Source Project: bigdata-tutorial   Source File: HBaseDDLHandlerTest.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	String quorum = "192.168.0.30,192.168.0.31,192.168.0.32";
	//quorum = "192.168.8.191,192.168.1.192,192.168.1.193";
	int port = 2181;
	String znode = "/hyperbase1";
	HBaseConnPool connPool = new HBaseClientManager(quorum, port, znode);
	HBaseDDLHandler ddlHandler = new HBaseDDLHandler(connPool);

	String tableName = "demo_test";
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	String columnFamily = "cf";
	System.out.println("=============================== : create");
	ddlHandler.createTable(tableName, columnFamily, "cf2");

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : alter");
	HBaseAdmin admin = new HBaseAdmin(connPool.getConn());
	admin.disableTable(tableName);
	HTableInterface htable = ddlHandler.getTable(tableName);
	HTableDescriptor tableDesc = admin.getTableDescriptor(htable.getTableName());
	tableDesc.removeFamily(Bytes.toBytes("cf2"));
	HColumnDescriptor newhcd = new HColumnDescriptor("cf3");
	newhcd.setMaxVersions(2);
	newhcd.setKeepDeletedCells(KeepDeletedCells.TRUE);
	tableDesc.addFamily(newhcd);

	admin.modifyTable(tableName, tableDesc);
	admin.enableTable(tableName);
	admin.close();

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	connPool.closeConn();
}
 
Example 11
Source Project: hbase   Source File: ThriftUtilities.java    License: Apache License 2.0 5 votes vote down vote up
public static KeepDeletedCells keepDeletedCellsFromThrift(TKeepDeletedCells in) {
  switch (in.getValue()) {
    case 0: return KeepDeletedCells.FALSE;
    case 1: return KeepDeletedCells.TRUE;
    case 2: return KeepDeletedCells.TTL;
    default: return KeepDeletedCells.FALSE;
  }
}
 
Example 12
Source Project: hbase   Source File: ThriftUtilities.java    License: Apache License 2.0 5 votes vote down vote up
public static TKeepDeletedCells keepDeletedCellsFromHBase(KeepDeletedCells in) {
  switch (in) {
    case FALSE: return TKeepDeletedCells.FALSE;
    case TRUE: return TKeepDeletedCells.TRUE;
    case TTL: return TKeepDeletedCells.TTL;
    default: return TKeepDeletedCells.FALSE;
  }
}
 
Example 13
Source Project: hbase   Source File: NormalUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
protected NormalUserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns,
    boolean hasNullColumn, DeleteTracker deletes, long oldestUnexpiredTS, long now) {
  super(scan, scanInfo, columns, hasNullColumn, oldestUnexpiredTS, now);
  this.deletes = deletes;
  this.get = scan.isGetScan();
  this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE;
}
 
Example 14
Source Project: hbase   Source File: CompactionScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
protected final void trackDelete(Cell cell) {
  // If keepDeletedCells is true, then we only remove cells by versions or TTL during
  // compaction, so we do not need to track delete here.
  // If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
  // minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
  // need to track the delete marker to see if it masks some cells.
  if (keepDeletedCells == KeepDeletedCells.FALSE
      || (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
    deletes.add(cell);
  }
}
 
Example 15
Source Project: hbase   Source File: TestRegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
private ScanInfo getScanInfo() {
  int oldMaxVersions = 1;
  int oldMinVersions = 0;
  long oldTTL = 10000;

  return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
  KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
    CellComparator.getInstance(), true);
}
 
Example 16
Source Project: hbase   Source File: TestUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This is a cryptic test. It is checking that we don't include a fake cell, one that has a
 * timestamp of {@link HConstants#OLDEST_TIMESTAMP}. See HBASE-16074 for background.
 * @throws IOException
 */
@Test
public void testNeverIncludeFakeCell() throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Do with fam2 which has a col2 qualifier.
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
    new ScanInfo(this.conf, fam2, 10, 1, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);
  Cell kv = new KeyValue(row1, fam2, col2, 1, data);
  Cell cell = PrivateCellUtil.createLastOnRowCol(kv);
  qm.setToNewRow(kv);
  MatchCode code = qm.match(cell);
  assertFalse(code.compareTo(MatchCode.SEEK_NEXT_COL) != 0);
}
 
Example 17
Source Project: hbase   Source File: TestUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
 * early from the row (skipping later non-expired KeyValues). This version mimics a Get with
 * explicitly specified column qualifiers.
 * @throws IOException
 */
@Test
public void testMatch_ExpiredExplicit() throws IOException {

  long testTTL = 1000;
  MatchCode[] expected = new MatchCode[] { ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW, ScanQueryMatcher.MatchCode.DONE };

  long now = EnvironmentEdgeManager.currentTime();
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
    new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - testTTL, now, null);

  KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
      new KeyValue(row1, fam2, col2, now - 50, data),
      new KeyValue(row1, fam2, col3, now - 5000, data),
      new KeyValue(row1, fam2, col4, now - 500, data),
      new KeyValue(row1, fam2, col5, now - 10000, data),
      new KeyValue(row2, fam1, col1, now - 10, data) };

  KeyValue k = kvs[0];
  qm.setToNewRow(k);

  List<MatchCode> actual = new ArrayList<>(kvs.length);
  for (KeyValue kv : kvs) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example 18
Source Project: hbase   Source File: TestUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
 * early from the row (skipping later non-expired KeyValues). This version mimics a Get with
 * wildcard-inferred column qualifiers.
 * @throws IOException
 */
@Test
public void testMatch_ExpiredWildcard() throws IOException {

  long testTTL = 1000;
  MatchCode[] expected =
      new MatchCode[] { ScanQueryMatcher.MatchCode.INCLUDE, ScanQueryMatcher.MatchCode.INCLUDE,
        ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.INCLUDE,
        ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.DONE };

  long now = EnvironmentEdgeManager.currentTime();
  UserScanQueryMatcher qm =
      UserScanQueryMatcher.create(scan,
        new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
            HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
        null, now - testTTL, now, null);

  KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
      new KeyValue(row1, fam2, col2, now - 50, data),
      new KeyValue(row1, fam2, col3, now - 5000, data),
      new KeyValue(row1, fam2, col4, now - 500, data),
      new KeyValue(row1, fam2, col5, now - 10000, data),
      new KeyValue(row2, fam1, col1, now - 10, data) };
  KeyValue k = kvs[0];
  qm.setToNewRow(k);

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(kvs.length);
  for (KeyValue kv : kvs) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example 19
Source Project: hbase   Source File: TestUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMatchWhenFilterReturnsIncludeAndSeekNextRow() throws IOException {
  List<MatchCode> expected = new ArrayList<>();
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
  expected.add(ScanQueryMatcher.MatchCode.DONE);

  Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeAndSeekNextRowFilter());

  long now = EnvironmentEdgeManager.currentTime();

  // scan with column 2,4,5
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(
    scanWithFilter, new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);

  List<KeyValue> memstore = new ArrayList<>();
  // ColumnTracker will return INCLUDE_AND_SEEK_NEXT_COL , and filter will return
  // INCLUDE_AND_SEEK_NEXT_ROW, so final match code will be INCLUDE_AND_SEEK_NEXT_ROW.
  memstore.add(new KeyValue(row1, fam2, col2, 1, data));
  memstore.add(new KeyValue(row2, fam1, col1, data));

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(memstore.size());
  KeyValue k = memstore.get(0);
  qm.setToNewRow(k);

  for (KeyValue kv : memstore) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.size(), actual.size());
  for (int i = 0; i < expected.size(); i++) {
    LOG.debug("expected " + expected.get(i) + ", actual " + actual.get(i));
    assertEquals(expected.get(i), actual.get(i));
  }
}
 
Example 20
Source Project: hbase   Source File: TestUserScanQueryMatcher.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Here is the unit test for UserScanQueryMatcher#mergeFilterResponse: the match code may be
 * changed to SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL after merging with filterResponse, even
 * if the passed match code is neither SEEK_NEXT_COL nor INCLUDE_AND_SEEK_NEXT_COL. In that case,
 * we need to make sure that the ColumnTracker has been switched to the next column. <br/>
 * An effective test way is: we only need to check the cell from getKeyForNextColumn(). because
 * that as long as the UserScanQueryMatcher returns SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL,
 * UserScanQueryMatcher#getKeyForNextColumn should return an cell whose column is larger than the
 * current cell's.
 */
@Test
public void testMergeFilterResponseCase2() throws Exception {
  List<MatchCode> expected = new ArrayList<>();
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);

  Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeFilter()).readVersions(3);

  long now = EnvironmentEdgeManager.currentTime();

  // scan with column 2,4,5, the family with maxVersion = 5
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(
    scanWithFilter, new ScanInfo(this.conf, fam2, 0, 5, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);

  List<KeyValue> memstore = new ArrayList<>();

  memstore.add(new KeyValue(row1, fam1, col2, 1, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 2, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 3, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 4, data)); // match code will be SEEK_NEXT_COL

  KeyValue k = memstore.get(0);
  qm.setToNewRow(k);

  for (int i = 0; i < memstore.size(); i++) {
    assertEquals(expected.get(i), qm.match(memstore.get(i)));
  }

  // For last cell, the query matcher will return SEEK_NEXT_COL, and the
  // ColumnTracker will skip to the next column, which is col4.
  Cell lastCell = memstore.get(memstore.size() - 1);
  Cell nextCell = qm.getKeyForNextColumn(lastCell);
  assertArrayEquals(nextCell.getQualifierArray(), col4);
}
 
Example 21
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(2, results.size());
    assertEquals(kvs[1], results.get(0));
    assertEquals(kvs[2], results.get(1));
    results.clear();

    assertEquals(true, scanner.next(results));
    assertEquals(3, results.size());
    assertEquals(kvs[4], results.get(0));
    assertEquals(kvs[5], results.get(1));
    assertEquals(kvs[6], results.get(2));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 22
Source Project: hbase   Source File: TestStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
    ScanType.COMPACT_RETAIN_DELETES, scanners)) {
    assertFalse(storeScanner.isScanUsePread());
  }
}
 
Example 23
Source Project: hbase   Source File: ColumnFamilyDescriptorBuilder.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Retain all versions for a given TTL(retentionInterval), and then only a specific number
 * of versions(versionAfterInterval) after that interval elapses.
 *
 * @param retentionInterval Retain all versions for this interval
 * @param versionAfterInterval Retain no of versions to retain after retentionInterval
 * @return this (for chained invocation)
 */
public ModifyableColumnFamilyDescriptor setVersionsWithTimeToLive(
    final int retentionInterval, final int versionAfterInterval) {
  ModifyableColumnFamilyDescriptor modifyableColumnFamilyDescriptor =
    setVersions(versionAfterInterval, Integer.MAX_VALUE);
  modifyableColumnFamilyDescriptor.setTimeToLive(retentionInterval);
  modifyableColumnFamilyDescriptor.setKeepDeletedCells(KeepDeletedCells.TTL);
  return modifyableColumnFamilyDescriptor;
}
 
Example 24
Source Project: phoenix   Source File: DropMetadataIT.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDropViewKeepsHTable() throws Exception {
    Connection conn = getConnection();
    Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
    String hbaseNativeViewName = generateUniqueName();

    byte[] hbaseNativeBytes = SchemaUtil.getTableNameAsBytes(HBASE_NATIVE_SCHEMA_NAME, hbaseNativeViewName);
    try {
         TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(hbaseNativeBytes));
        ColumnFamilyDescriptor columnDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_NAME)
                .setKeepDeletedCells(KeepDeletedCells.TRUE).build();
        builder.addColumnFamily(columnDescriptor);
        admin.createTable(builder.build());
    } finally {
        admin.close();
    }
    
    conn.createStatement().execute("create view " + hbaseNativeViewName+
            "   (uint_key unsigned_int not null," +
            "    ulong_key unsigned_long not null," +
            "    string_key varchar not null,\n" +
            "    \"1\".uint_col unsigned_int," +
            "    \"1\".ulong_col unsigned_long" +
            "    CONSTRAINT pk PRIMARY KEY (uint_key, ulong_key, string_key))\n" +
            ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING + "='" + DataBlockEncoding.NONE + "'");
    conn.createStatement().execute("drop view " + hbaseNativeViewName);
    conn.close();
}
 
Example 25
Source Project: phoenix   Source File: TestIndexManagementUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param admin to create the table
 * @param index descriptor to update before creating table
 */
public static void createIndexTable(Admin admin, TableDescriptorBuilder indexBuilder) throws IOException {
      indexBuilder.addColumnFamily(
              ColumnFamilyDescriptorBuilder.newBuilder(CoveredColumnIndexCodec.INDEX_ROW_COLUMN_FAMILY)
                      .setKeepDeletedCells(KeepDeletedCells.TRUE).build());
  admin.createTable(indexBuilder.build());
}
 
Example 26
Source Project: phoenix   Source File: AlterTableIT.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testSetHTableHColumnAndPhoenixTableProperties() throws Exception {
    Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
    String ddl = "CREATE TABLE T3 (\n"
            +"ID1 VARCHAR(15) NOT NULL,\n"
            +"ID2 VARCHAR(15) NOT NULL,\n"
            +"CREATED_DATE DATE,\n"
            +"CF1.CREATION_TIME BIGINT,\n"
            +"CF2.LAST_USED DATE,\n"
            +"CONSTRAINT PK PRIMARY KEY (ID1, ID2)) IMMUTABLE_ROWS=true";
    Connection conn = DriverManager.getConnection(getUrl(), props);
    conn.createStatement().execute(ddl);
    assertImmutableRows(conn, "T3", true);
    ddl = "ALTER TABLE T3 SET COMPACTION_ENABLED = FALSE, VERSIONS = 10";
    conn.createStatement().execute(ddl);
    ddl = "ALTER TABLE T3 SET COMPACTION_ENABLED = FALSE, CF1.MIN_VERSIONS = 1, CF2.MIN_VERSIONS = 3, MIN_VERSIONS = 8, IMMUTABLE_ROWS=false, CF1.KEEP_DELETED_CELLS = true, KEEP_DELETED_CELLS = false";
    conn.createStatement().execute(ddl);
    assertImmutableRows(conn, "T3", false);

    try (HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
        HTableDescriptor tableDesc = admin.getTableDescriptor(Bytes.toBytes("T3"));
        HColumnDescriptor[] columnFamilies = tableDesc.getColumnFamilies();
        assertEquals(3, columnFamilies.length);

        assertEquals("0", columnFamilies[0].getNameAsString());
        assertEquals(8, columnFamilies[0].getMinVersions());
        assertEquals(10, columnFamilies[0].getMaxVersions());
        assertEquals(KeepDeletedCells.FALSE, columnFamilies[0].getKeepDeletedCellsAsEnum());

        assertEquals("CF1", columnFamilies[1].getNameAsString());
        assertEquals(1, columnFamilies[1].getMinVersions());
        assertEquals(10, columnFamilies[1].getMaxVersions());
        assertEquals(KeepDeletedCells.TRUE, columnFamilies[1].getKeepDeletedCellsAsEnum());

        assertEquals("CF2", columnFamilies[2].getNameAsString());
        assertEquals(3, columnFamilies[2].getMinVersions());
        assertEquals(10, columnFamilies[2].getMaxVersions());
        assertEquals(KeepDeletedCells.FALSE, columnFamilies[2].getKeepDeletedCellsAsEnum());

        assertEquals(Boolean.toString(false), tableDesc.getValue(HTableDescriptor.COMPACTION_ENABLED));
    }
}
 
Example 27
Source Project: hbase   Source File: CustomizedScanInfoBuilder.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setKeepDeletedCells(KeepDeletedCells keepDeletedCells) {
  this.keepDeletedCells = keepDeletedCells;
}
 
Example 28
Source Project: hbase   Source File: CustomizedScanInfoBuilder.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public KeepDeletedCells getKeepDeletedCells() {
  return keepDeletedCells != null ? keepDeletedCells : scanInfo.getKeepDeletedCells();
}
 
Example 29
Source Project: hbase   Source File: ScanInfo.java    License: Apache License 2.0 4 votes vote down vote up
public KeepDeletedCells getKeepDeletedCells() {
  return keepDeletedCells;
}
 
Example 30
Source Project: hbase   Source File: ScanInfo.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Used for CP users for customizing max versions, ttl and keepDeletedCells.
 */
ScanInfo customize(int maxVersions, long ttl, KeepDeletedCells keepDeletedCells) {
  return customize(maxVersions, ttl, keepDeletedCells, minVersions);
}