org.apache.hadoop.hbase.KeepDeletedCells Java Examples

The following examples show how to use org.apache.hadoop.hbase.KeepDeletedCells. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DropDeletesCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final MatchCode tryDropDelete(Cell cell) {
  long timestamp = cell.getTimestamp();
  // If it is not the time to drop the delete marker, just return
  if (timeToPurgeDeletes > 0 && now - timestamp <= timeToPurgeDeletes) {
    return MatchCode.INCLUDE;
  }
  if (keepDeletedCells == KeepDeletedCells.TRUE
      || (keepDeletedCells == KeepDeletedCells.TTL && timestamp >= oldestUnexpiredTS)) {
    // If keepDeletedCell is true, or the delete marker is not expired yet, we should include it
    // in version counting to see if we can drop it. The only exception is that, we can make
    // sure that no put is older than this delete marker. And under this situation, all later
    // cells of this column(must be delete markers) can be skipped.
    if (timestamp < earliestPutTs) {
      return columns.getNextRowOrNextColumn(cell);
    } else {
      return null;
    }
  } else {
    return MatchCode.SKIP;
  }
}
 
Example #2
Source File: TestKeepDeletes.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * The ExplicitColumnTracker does not support "raw" scanning.
 */
@Test
public void testRawScanWithColumns() throws Exception {
  HTableDescriptor htd = hbu.createTableDescriptor(TableName.valueOf(name.getMethodName()), 0, 3,
      HConstants.FOREVER, KeepDeletedCells.TRUE);
  Region region = hbu.createLocalHRegion(htd, null, null);

  Scan s = new Scan();
  s.setRaw(true);
  s.readAllVersions();
  s.addColumn(c0, c0);

  try {
    region.getScanner(s);
    fail("raw scanner with columns should have failed");
  } catch (org.apache.hadoop.hbase.DoNotRetryIOException dnre) {
    // ok!
  }

  HBaseTestingUtility.closeRegionAndWAL(region);
}
 
Example #3
Source File: TestCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example #4
Source File: ScanInfo.java    From hbase with Apache License 2.0 6 votes vote down vote up
private ScanInfo(byte[] family, int minVersions, int maxVersions, long ttl,
    KeepDeletedCells keepDeletedCells, long timeToPurgeDeletes, CellComparator comparator,
    long tableMaxRowSize, boolean usePread, long cellsPerTimeoutCheck,
    boolean parallelSeekEnabled, long preadMaxBytes, boolean newVersionBehavior) {
  this.family = family;
  this.minVersions = minVersions;
  this.maxVersions = maxVersions;
  this.ttl = ttl;
  this.keepDeletedCells = keepDeletedCells;
  this.timeToPurgeDeletes = timeToPurgeDeletes;
  this.comparator = comparator;
  this.tableMaxRowSize = tableMaxRowSize;
  this.usePread = usePread;
  this.cellsPerTimeoutCheck = cellsPerTimeoutCheck;
  this.parallelSeekEnabled = parallelSeekEnabled;
  this.preadMaxBytes = preadMaxBytes;
  this.newVersionBehavior = newVersionBehavior;
}
 
Example #5
Source File: TestMinVersions.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testMinVersionsWithKeepDeletedCellsTTL() throws Exception {
  int ttl = 4;
  ColumnFamilyDescriptor cfd =
    ColumnFamilyDescriptorBuilder.newBuilder(c0)
      .setVersionsWithTimeToLive(ttl, 2).build();
  verifyVersionedCellKeyValues(ttl, cfd);

  cfd = ColumnFamilyDescriptorBuilder.newBuilder(c0)
    .setMinVersions(2)
    .setMaxVersions(Integer.MAX_VALUE)
    .setTimeToLive(ttl)
    .setKeepDeletedCells(KeepDeletedCells.TTL)
    .build();
  verifyVersionedCellKeyValues(ttl, cfd);
}
 
Example #6
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example #7
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadVersionWithRawAndFilter() throws IOException {
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
          , CellComparator.getInstance(), false);
  KeyValue [] kvs = new KeyValue[] {
          create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[]{
      new KeyValueScanFixture(CellComparator.getInstance(), kvs)
    });

  BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
  Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
  scanSpec.setFilter(filter);
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(2, results.size());
  }
}
 
Example #8
Source File: PropertiesInSyncIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method to verify HBase column family properties
 * @param tableName Physical HBase table whose properties are to be verified
 * @param conn Phoenix connection
 * @param propModified true if we have altered any of the properties to be kept in sync, false otherwise
 * @param ignoreTTL We cannot modfiy a table level property when adding a column, so in those cases,
 *                 ignore the check for TTL modification
 * @throws Exception
 */
private void verifyHBaseColumnFamilyProperties(String tableName, Connection conn, boolean propModified,
        boolean ignoreTTL) throws Exception {
    final int expectedTTL = propModified ? MODIFIED_TTL_VALUE:INITIAL_TTL_VALUE;
    final KeepDeletedCells expectedKeepDeletedCells = propModified ? MODIFIED_KEEP_DELETED_CELLS_VALUE: INITIAL_KEEP_DELETED_CELLS_VALUE;
    final int expectedReplicationScope = propModified ? MODIFIED_REPLICATION_SCOPE_VALUE:INITIAL_REPLICATION_SCOPE_VALUE;

    try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
        // Note that this includes the local index column family as well
        ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(tableName))
                .getColumnFamilies();
        for (ColumnFamilyDescriptor cfd: columnFamilies) {
            if (!ignoreTTL) {
                assertEquals("Mismatch in TTL", expectedTTL, cfd.getTimeToLive());
            }
            assertEquals("Mismatch in KEEP_DELETED_CELLS", expectedKeepDeletedCells, cfd.getKeepDeletedCells());
            assertEquals("Mismatch in REPLICATION_SCOPE", expectedReplicationScope, cfd.getScope());
        }
    }
}
 
Example #9
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
    ScanType.COMPACT_RETAIN_DELETES, scanners)) {
    assertFalse(storeScanner.isScanUsePread());
  }
}
 
Example #10
Source File: TestStoreScanner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(2, results.size());
    assertEquals(kvs[1], results.get(0));
    assertEquals(kvs[2], results.get(1));
    results.clear();

    assertEquals(true, scanner.next(results));
    assertEquals(3, results.size());
    assertEquals(kvs[4], results.get(0));
    assertEquals(kvs[5], results.get(1));
    assertEquals(kvs[6], results.get(2));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example #11
Source File: HBaseDDLHandlerTest.java    From bigdata-tutorial with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	String quorum = "192.168.0.30,192.168.0.31,192.168.0.32";
	//quorum = "192.168.8.191,192.168.1.192,192.168.1.193";
	int port = 2181;
	String znode = "/hyperbase1";
	HBaseConnPool connPool = new HBaseClientManager(quorum, port, znode);
	HBaseDDLHandler ddlHandler = new HBaseDDLHandler(connPool);

	String tableName = "demo_test";
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	String columnFamily = "cf";
	System.out.println("=============================== : create");
	ddlHandler.createTable(tableName, columnFamily, "cf2");

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : alter");
	HBaseAdmin admin = new HBaseAdmin(connPool.getConn());
	admin.disableTable(tableName);
	HTableInterface htable = ddlHandler.getTable(tableName);
	HTableDescriptor tableDesc = admin.getTableDescriptor(htable.getTableName());
	tableDesc.removeFamily(Bytes.toBytes("cf2"));
	HColumnDescriptor newhcd = new HColumnDescriptor("cf3");
	newhcd.setMaxVersions(2);
	newhcd.setKeepDeletedCells(KeepDeletedCells.TRUE);
	tableDesc.addFamily(newhcd);

	admin.modifyTable(tableName, tableDesc);
	admin.enableTable(tableName);
	admin.close();

	System.out.println("=============================== : desc");
	HBaseUtils.printTableInfo(ddlHandler.getTable(tableName));
	System.out.println("=============================== : delete");
	ddlHandler.deleteTable(tableName);

	connPool.closeConn();
}
 
Example #12
Source File: ColumnFamilyDescriptorBuilder.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Retain all versions for a given TTL(retentionInterval), and then only a specific number
 * of versions(versionAfterInterval) after that interval elapses.
 *
 * @param retentionInterval Retain all versions for this interval
 * @param versionAfterInterval Retain no of versions to retain after retentionInterval
 * @return this (for chained invocation)
 */
public ModifyableColumnFamilyDescriptor setVersionsWithTimeToLive(
    final int retentionInterval, final int versionAfterInterval) {
  ModifyableColumnFamilyDescriptor modifyableColumnFamilyDescriptor =
    setVersions(versionAfterInterval, Integer.MAX_VALUE);
  modifyableColumnFamilyDescriptor.setTimeToLive(retentionInterval);
  modifyableColumnFamilyDescriptor.setKeepDeletedCells(KeepDeletedCells.TTL);
  return modifyableColumnFamilyDescriptor;
}
 
Example #13
Source File: DropMetadataIT.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testDropViewKeepsHTable() throws Exception {
    Connection conn = getConnection();
    Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
    String hbaseNativeViewName = generateUniqueName();

    byte[] hbaseNativeBytes = SchemaUtil.getTableNameAsBytes(HBASE_NATIVE_SCHEMA_NAME, hbaseNativeViewName);
    try {
         TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(TableName.valueOf(hbaseNativeBytes));
        ColumnFamilyDescriptor columnDescriptor = ColumnFamilyDescriptorBuilder.newBuilder(FAMILY_NAME)
                .setKeepDeletedCells(KeepDeletedCells.TRUE).build();
        builder.addColumnFamily(columnDescriptor);
        admin.createTable(builder.build());
    } finally {
        admin.close();
    }
    
    conn.createStatement().execute("create view " + hbaseNativeViewName+
            "   (uint_key unsigned_int not null," +
            "    ulong_key unsigned_long not null," +
            "    string_key varchar not null,\n" +
            "    \"1\".uint_col unsigned_int," +
            "    \"1\".ulong_col unsigned_long" +
            "    CONSTRAINT pk PRIMARY KEY (uint_key, ulong_key, string_key))\n" +
            ColumnFamilyDescriptorBuilder.DATA_BLOCK_ENCODING + "='" + DataBlockEncoding.NONE + "'");
    conn.createStatement().execute("drop view " + hbaseNativeViewName);
    conn.close();
}
 
Example #14
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Here is the unit test for UserScanQueryMatcher#mergeFilterResponse: the match code may be
 * changed to SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL after merging with filterResponse, even
 * if the passed match code is neither SEEK_NEXT_COL nor INCLUDE_AND_SEEK_NEXT_COL. In that case,
 * we need to make sure that the ColumnTracker has been switched to the next column. <br/>
 * An effective test way is: we only need to check the cell from getKeyForNextColumn(). because
 * that as long as the UserScanQueryMatcher returns SEEK_NEXT_COL or INCLUDE_AND_SEEK_NEXT_COL,
 * UserScanQueryMatcher#getKeyForNextColumn should return an cell whose column is larger than the
 * current cell's.
 */
@Test
public void testMergeFilterResponseCase2() throws Exception {
  List<MatchCode> expected = new ArrayList<>();
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);

  Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeFilter()).readVersions(3);

  long now = EnvironmentEdgeManager.currentTime();

  // scan with column 2,4,5, the family with maxVersion = 5
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(
    scanWithFilter, new ScanInfo(this.conf, fam2, 0, 5, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);

  List<KeyValue> memstore = new ArrayList<>();

  memstore.add(new KeyValue(row1, fam1, col2, 1, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 2, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 3, data)); // match code will be INCLUDE
  memstore.add(new KeyValue(row1, fam1, col2, 4, data)); // match code will be SEEK_NEXT_COL

  KeyValue k = memstore.get(0);
  qm.setToNewRow(k);

  for (int i = 0; i < memstore.size(); i++) {
    assertEquals(expected.get(i), qm.match(memstore.get(i)));
  }

  // For last cell, the query matcher will return SEEK_NEXT_COL, and the
  // ColumnTracker will skip to the next column, which is col4.
  Cell lastCell = memstore.get(memstore.size() - 1);
  Cell nextCell = qm.getKeyForNextColumn(lastCell);
  assertArrayEquals(nextCell.getQualifierArray(), col4);
}
 
Example #15
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testMatchWhenFilterReturnsIncludeAndSeekNextRow() throws IOException {
  List<MatchCode> expected = new ArrayList<>();
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_ROW);
  expected.add(ScanQueryMatcher.MatchCode.DONE);

  Scan scanWithFilter = new Scan(scan).setFilter(new AlwaysIncludeAndSeekNextRowFilter());

  long now = EnvironmentEdgeManager.currentTime();

  // scan with column 2,4,5
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(
    scanWithFilter, new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);

  List<KeyValue> memstore = new ArrayList<>();
  // ColumnTracker will return INCLUDE_AND_SEEK_NEXT_COL , and filter will return
  // INCLUDE_AND_SEEK_NEXT_ROW, so final match code will be INCLUDE_AND_SEEK_NEXT_ROW.
  memstore.add(new KeyValue(row1, fam2, col2, 1, data));
  memstore.add(new KeyValue(row2, fam1, col1, data));

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(memstore.size());
  KeyValue k = memstore.get(0);
  qm.setToNewRow(k);

  for (KeyValue kv : memstore) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.size(), actual.size());
  for (int i = 0; i < expected.size(); i++) {
    LOG.debug("expected " + expected.get(i) + ", actual " + actual.get(i));
    assertEquals(expected.get(i), actual.get(i));
  }
}
 
Example #16
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
 * early from the row (skipping later non-expired KeyValues). This version mimics a Get with
 * wildcard-inferred column qualifiers.
 * @throws IOException
 */
@Test
public void testMatch_ExpiredWildcard() throws IOException {

  long testTTL = 1000;
  MatchCode[] expected =
      new MatchCode[] { ScanQueryMatcher.MatchCode.INCLUDE, ScanQueryMatcher.MatchCode.INCLUDE,
        ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.INCLUDE,
        ScanQueryMatcher.MatchCode.SEEK_NEXT_COL, ScanQueryMatcher.MatchCode.DONE };

  long now = EnvironmentEdgeManager.currentTime();
  UserScanQueryMatcher qm =
      UserScanQueryMatcher.create(scan,
        new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
            HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
        null, now - testTTL, now, null);

  KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
      new KeyValue(row1, fam2, col2, now - 50, data),
      new KeyValue(row1, fam2, col3, now - 5000, data),
      new KeyValue(row1, fam2, col4, now - 500, data),
      new KeyValue(row1, fam2, col5, now - 10000, data),
      new KeyValue(row2, fam1, col1, now - 10, data) };
  KeyValue k = kvs[0];
  qm.setToNewRow(k);

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(kvs.length);
  for (KeyValue kv : kvs) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example #17
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Verify that {@link ScanQueryMatcher} only skips expired KeyValue instances and does not exit
 * early from the row (skipping later non-expired KeyValues). This version mimics a Get with
 * explicitly specified column qualifiers.
 * @throws IOException
 */
@Test
public void testMatch_ExpiredExplicit() throws IOException {

  long testTTL = 1000;
  MatchCode[] expected = new MatchCode[] { ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.INCLUDE_AND_SEEK_NEXT_COL,
      ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW, ScanQueryMatcher.MatchCode.DONE };

  long now = EnvironmentEdgeManager.currentTime();
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
    new ScanInfo(this.conf, fam2, 0, 1, testTTL, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - testTTL, now, null);

  KeyValue[] kvs = new KeyValue[] { new KeyValue(row1, fam2, col1, now - 100, data),
      new KeyValue(row1, fam2, col2, now - 50, data),
      new KeyValue(row1, fam2, col3, now - 5000, data),
      new KeyValue(row1, fam2, col4, now - 500, data),
      new KeyValue(row1, fam2, col5, now - 10000, data),
      new KeyValue(row2, fam1, col1, now - 10, data) };

  KeyValue k = kvs[0];
  qm.setToNewRow(k);

  List<MatchCode> actual = new ArrayList<>(kvs.length);
  for (KeyValue kv : kvs) {
    actual.add(qm.match(kv));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example #18
Source File: TestUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * This is a cryptic test. It is checking that we don't include a fake cell, one that has a
 * timestamp of {@link HConstants#OLDEST_TIMESTAMP}. See HBASE-16074 for background.
 * @throws IOException
 */
@Test
public void testNeverIncludeFakeCell() throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Do with fam2 which has a col2 qualifier.
  UserScanQueryMatcher qm = UserScanQueryMatcher.create(scan,
    new ScanInfo(this.conf, fam2, 10, 1, ttl, KeepDeletedCells.FALSE,
        HConstants.DEFAULT_BLOCKSIZE, 0, rowComparator, false),
    get.getFamilyMap().get(fam2), now - ttl, now, null);
  Cell kv = new KeyValue(row1, fam2, col2, 1, data);
  Cell cell = PrivateCellUtil.createLastOnRowCol(kv);
  qm.setToNewRow(kv);
  MatchCode code = qm.match(cell);
  assertFalse(code.compareTo(MatchCode.SEEK_NEXT_COL) != 0);
}
 
Example #19
Source File: TestIndexManagementUtil.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * @param admin to create the table
 * @param index descriptor to update before creating table
 */
public static void createIndexTable(Admin admin, TableDescriptorBuilder indexBuilder) throws IOException {
      indexBuilder.addColumnFamily(
              ColumnFamilyDescriptorBuilder.newBuilder(CoveredColumnIndexCodec.INDEX_ROW_COLUMN_FAMILY)
                      .setKeepDeletedCells(KeepDeletedCells.TRUE).build());
  admin.createTable(indexBuilder.build());
}
 
Example #20
Source File: HalyardTableUtils.java    From Halyard with Apache License 2.0 5 votes vote down vote up
private static HColumnDescriptor createColumnFamily() {
    return new HColumnDescriptor(CF_NAME)
            .setMaxVersions(1)
            .setBlockCacheEnabled(true)
            .setBloomFilterType(BloomType.ROW)
            .setCompressionType(DEFAULT_COMPRESSION_ALGORITHM)
            .setDataBlockEncoding(DEFAULT_DATABLOCK_ENCODING)
            .setCacheBloomsOnWrite(true)
            .setCacheDataOnWrite(true)
            .setCacheIndexesOnWrite(true)
            .setKeepDeletedCells(KeepDeletedCells.FALSE)
            .setValue(HTableDescriptor.MAX_FILESIZE, REGION_MAX_FILESIZE)
            .setValue(HTableDescriptor.SPLIT_POLICY, REGION_SPLIT_POLICY);
}
 
Example #21
Source File: TestRegionCoprocessorHost.java    From hbase with Apache License 2.0 5 votes vote down vote up
private ScanInfo getScanInfo() {
  int oldMaxVersions = 1;
  int oldMinVersions = 0;
  long oldTTL = 10000;

  return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
  KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
    CellComparator.getInstance(), true);
}
 
Example #22
Source File: CompactionScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected final void trackDelete(Cell cell) {
  // If keepDeletedCells is true, then we only remove cells by versions or TTL during
  // compaction, so we do not need to track delete here.
  // If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
  // minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
  // need to track the delete marker to see if it masks some cells.
  if (keepDeletedCells == KeepDeletedCells.FALSE
      || (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
    deletes.add(cell);
  }
}
 
Example #23
Source File: NormalUserScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected NormalUserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns,
    boolean hasNullColumn, DeleteTracker deletes, long oldestUnexpiredTS, long now) {
  super(scan, scanInfo, columns, hasNullColumn, oldestUnexpiredTS, now);
  this.deletes = deletes;
  this.get = scan.isGetScan();
  this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE;
}
 
Example #24
Source File: ThriftUtilities.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static TKeepDeletedCells keepDeletedCellsFromHBase(KeepDeletedCells in) {
  switch (in) {
    case FALSE: return TKeepDeletedCells.FALSE;
    case TRUE: return TKeepDeletedCells.TRUE;
    case TTL: return TKeepDeletedCells.TTL;
    default: return TKeepDeletedCells.FALSE;
  }
}
 
Example #25
Source File: ThriftUtilities.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static KeepDeletedCells keepDeletedCellsFromThrift(TKeepDeletedCells in) {
  switch (in.getValue()) {
    case 0: return KeepDeletedCells.FALSE;
    case 1: return KeepDeletedCells.TRUE;
    case 2: return KeepDeletedCells.TTL;
    default: return KeepDeletedCells.FALSE;
  }
}
 
Example #26
Source File: UngroupedAggregateRegionObserver.java    From phoenix with Apache License 2.0 4 votes vote down vote up
@Override
public void preCompactScannerOpen(final ObserverContext<RegionCoprocessorEnvironment> c, final Store store,
        ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
        final CompactionRequest request) throws IOException {
    // Compaction and split upcalls run with the effective user context of the requesting user.
    // This will lead to failure of cross cluster RPC if the effective user is not
    // the login user. Switch to the login user context to ensure we have the expected
    // security context.
    final String fullTableName = c.getEnvironment().getRegion().getRegionInfo().getTable().getNameAsString();
    // since we will make a call to syscat, do nothing if we are compacting syscat itself
    if (request.isMajor() && !PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME.equals(fullTableName)) {
        User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
            @Override
            public Void run() throws Exception {
                // If the index is disabled, keep the deleted cells so the rebuild doesn't corrupt the index
                try (PhoenixConnection conn =
                        QueryUtil.getConnectionOnServer(compactionConfig).unwrap(PhoenixConnection.class)) {
                    PTable table = PhoenixRuntime.getTableNoCache(conn, fullTableName);
                    List<PTable> indexes = PTableType.INDEX.equals(table.getType()) ? Lists.newArrayList(table) : table.getIndexes();
                    // FIXME need to handle views and indexes on views as well
                    for (PTable index : indexes) {
                        if (index.getIndexDisableTimestamp() != 0) {
                            LOGGER.info(
                                "Modifying major compaction scanner to retain deleted cells for a table with disabled index: "
                                        + fullTableName);
                            options.setKeepDeletedCells(KeepDeletedCells.TRUE);
                            options.readAllVersions();
                            options.setTTL(Long.MAX_VALUE);
                        }
                    }
                } catch (Exception e) {
                    if (e instanceof TableNotFoundException) {
                        LOGGER.debug("Ignoring HBase table that is not a Phoenix table: " + fullTableName);
                        // non-Phoenix HBase tables won't be found, do nothing
                    } else {
                        LOGGER.error("Unable to modify compaction scanner to retain deleted cells for a table with disabled Index; "
                                + fullTableName,
                                e);
                    }
                }
                return null;
            }
        });
    }
}
 
Example #27
Source File: TestColumnFamilyDescriptorBuilder.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testBuilder() throws DeserializationException {
  ColumnFamilyDescriptorBuilder builder
    = ColumnFamilyDescriptorBuilder.newBuilder(HConstants.CATALOG_FAMILY)
          .setInMemory(true)
          .setScope(HConstants.REPLICATION_SCOPE_LOCAL)
          .setBloomFilterType(BloomType.NONE);
  final int v = 123;
  builder.setBlocksize(v);
  builder.setTimeToLive(v);
  builder.setBlockCacheEnabled(!HColumnDescriptor.DEFAULT_BLOCKCACHE);
  builder.setValue(Bytes.toBytes("a"), Bytes.toBytes("b"));
  builder.setMaxVersions(v);
  assertEquals(v, builder.build().getMaxVersions());
  builder.setMinVersions(v);
  assertEquals(v, builder.build().getMinVersions());
  builder.setKeepDeletedCells(KeepDeletedCells.TRUE);
  builder.setInMemory(!HColumnDescriptor.DEFAULT_IN_MEMORY);
  boolean inmemory = builder.build().isInMemory();
  builder.setScope(v);
  builder.setDataBlockEncoding(DataBlockEncoding.FAST_DIFF);
  builder.setBloomFilterType(BloomType.ROW);
  builder.setCompressionType(Algorithm.SNAPPY);
  builder.setMobEnabled(true);
  builder.setMobThreshold(1000L);
  builder.setDFSReplication((short) v);

  ColumnFamilyDescriptor hcd = builder.build();
  byte [] bytes = ColumnFamilyDescriptorBuilder.toByteArray(hcd);
  ColumnFamilyDescriptor deserializedHcd = ColumnFamilyDescriptorBuilder.parseFrom(bytes);
  assertTrue(hcd.equals(deserializedHcd));
  assertEquals(v, hcd.getBlocksize());
  assertEquals(v, hcd.getTimeToLive());
  assertTrue(Bytes.equals(hcd.getValue(Bytes.toBytes("a")),
      deserializedHcd.getValue(Bytes.toBytes("a"))));
  assertEquals(hcd.getMaxVersions(), deserializedHcd.getMaxVersions());
  assertEquals(hcd.getMinVersions(), deserializedHcd.getMinVersions());
  assertEquals(hcd.getKeepDeletedCells(), deserializedHcd.getKeepDeletedCells());
  assertEquals(inmemory, deserializedHcd.isInMemory());
  assertEquals(hcd.getScope(), deserializedHcd.getScope());
  assertTrue(deserializedHcd.getCompressionType().equals(Compression.Algorithm.SNAPPY));
  assertTrue(deserializedHcd.getDataBlockEncoding().equals(DataBlockEncoding.FAST_DIFF));
  assertTrue(deserializedHcd.getBloomFilterType().equals(BloomType.ROW));
  assertEquals(hcd.isMobEnabled(), deserializedHcd.isMobEnabled());
  assertEquals(hcd.getMobThreshold(), deserializedHcd.getMobThreshold());
  assertEquals(v, deserializedHcd.getDFSReplication());
}
 
Example #28
Source File: ColumnFamilyDescriptorBuilder.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public KeepDeletedCells getKeepDeletedCells() {
  return getStringOrDefault(KEEP_DELETED_CELLS_BYTES,
      KeepDeletedCells::getValue, DEFAULT_KEEP_DELETED);
}
 
Example #29
Source File: ColumnFamilyDescriptorBuilder.java    From hbase with Apache License 2.0 4 votes vote down vote up
public ColumnFamilyDescriptorBuilder setKeepDeletedCells(KeepDeletedCells value) {
  desc.setKeepDeletedCells(value);
  return this;
}
 
Example #30
Source File: TestReversibleScanners.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Test
public void testReversibleStoreScanner() throws IOException {
  // write data to one memstore and two store files
  FileSystem fs = TEST_UTIL.getTestFileSystem();
  Path hfilePath = new Path(new Path(
      TEST_UTIL.getDataTestDir("testReversibleStoreScanner"), "regionname"),
      "familyname");
  CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
  HFileContextBuilder hcBuilder = new HFileContextBuilder();
  hcBuilder.withBlockSize(2 * 1024);
  HFileContext hFileContext = hcBuilder.build();
  StoreFileWriter writer1 = new StoreFileWriter.Builder(
      TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
      hfilePath).withFileContext(hFileContext).build();
  StoreFileWriter writer2 = new StoreFileWriter.Builder(
      TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
      hfilePath).withFileContext(hFileContext).build();

  MemStore memstore = new DefaultMemStore();
  writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1,
      writer2 });

  HStoreFile sf1 = new HStoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf,
      BloomType.NONE, true);

  HStoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf,
      BloomType.NONE, true);

  ScanInfo scanInfo =
      new ScanInfo(TEST_UTIL.getConfiguration(), FAMILYNAME, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparatorImpl.COMPARATOR, false);

  // Case 1.Test a full reversed scan
  Scan scan = new Scan();
  scan.setReversed(true);
  StoreScanner storeScanner =
      getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, QUALSIZE * ROWSIZE, ROWSIZE, false);

  // Case 2.Test reversed scan with a specified start row
  int startRowNum = ROWSIZE / 2;
  byte[] startRow = ROWS[startRowNum];
  scan.withStartRow(startRow);
  storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, QUALSIZE * (startRowNum + 1),
      startRowNum + 1, false);

  // Case 3.Test reversed scan with a specified start row and specified
  // qualifiers
  assertTrue(QUALSIZE > 2);
  scan.addColumn(FAMILYNAME, QUALS[0]);
  scan.addColumn(FAMILYNAME, QUALS[2]);
  storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, 2 * (startRowNum + 1), startRowNum + 1,
      false);

  // Case 4.Test reversed scan with mvcc based on case 3
  for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) {
    LOG.info("Setting read point to " + readPoint);
    storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, readPoint);
    int expectedRowCount = 0;
    int expectedKVCount = 0;
    for (int i = startRowNum; i >= 0; i--) {
      int kvCount = 0;
      if (makeMVCC(i, 0) <= readPoint) {
        kvCount++;
      }
      if (makeMVCC(i, 2) <= readPoint) {
        kvCount++;
      }
      if (kvCount > 0) {
        expectedRowCount++;
        expectedKVCount += kvCount;
      }
    }
    verifyCountAndOrder(storeScanner, expectedKVCount, expectedRowCount,
        false);
  }
}