Java Code Examples for org.apache.hadoop.hbase.KeepDeletedCells#FALSE

The following examples show how to use org.apache.hadoop.hbase.KeepDeletedCells#FALSE . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example 2
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 3
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReadVersionWithRawAndFilter() throws IOException {
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
          , CellComparator.getInstance(), false);
  KeyValue [] kvs = new KeyValue[] {
          create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[]{
      new KeyValueScanFixture(CellComparator.getInstance(), kvs)
    });

  BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
  Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
  scanSpec.setFilter(filter);
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(2, results.size());
  }
}
 
Example 4
Source Project: hbase   File: ThriftUtilities.java    License: Apache License 2.0 5 votes vote down vote up
public static KeepDeletedCells keepDeletedCellsFromThrift(TKeepDeletedCells in) {
  switch (in.getValue()) {
    case 0: return KeepDeletedCells.FALSE;
    case 1: return KeepDeletedCells.TRUE;
    case 2: return KeepDeletedCells.TTL;
    default: return KeepDeletedCells.FALSE;
  }
}
 
Example 5
protected NormalUserScanQueryMatcher(Scan scan, ScanInfo scanInfo, ColumnTracker columns,
    boolean hasNullColumn, DeleteTracker deletes, long oldestUnexpiredTS, long now) {
  super(scan, scanInfo, columns, hasNullColumn, oldestUnexpiredTS, now);
  this.deletes = deletes;
  this.get = scan.isGetScan();
  this.seePastDeleteMarkers = scanInfo.getKeepDeletedCells() != KeepDeletedCells.FALSE;
}
 
Example 6
protected final void trackDelete(Cell cell) {
  // If keepDeletedCells is true, then we only remove cells by versions or TTL during
  // compaction, so we do not need to track delete here.
  // If keepDeletedCells is TTL and the delete marker is expired, then we can make sure that the
  // minVerions is larger than 0(otherwise we will just return at preCheck). So here we still
  // need to track the delete marker to see if it masks some cells.
  if (keepDeletedCells == KeepDeletedCells.FALSE
      || (keepDeletedCells == KeepDeletedCells.TTL && cell.getTimestamp() < oldestUnexpiredTS)) {
    deletes.add(cell);
  }
}
 
Example 7
Source Project: hbase   File: TestRegionCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
private ScanInfo getScanInfo() {
  int oldMaxVersions = 1;
  int oldMinVersions = 0;
  long oldTTL = 10000;

  return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
  KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
    CellComparator.getInstance(), true);
}
 
Example 8
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(2, results.size());
    assertEquals(kvs[1], results.get(0));
    assertEquals(kvs[2], results.get(1));
    results.clear();

    assertEquals(true, scanner.next(results));
    assertEquals(3, results.size());
    assertEquals(kvs[4], results.get(0));
    assertEquals(kvs[5], results.get(1));
    assertEquals(kvs[6], results.get(2));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 
Example 9
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
    ScanType.COMPACT_RETAIN_DELETES, scanners)) {
    assertFalse(storeScanner.isScanUsePread());
  }
}
 
Example 10
Source Project: hbase   File: TestDefaultMemStore.java    License: Apache License 2.0 4 votes vote down vote up
/** Test getNextRow from memstore
 * @throws InterruptedException
 */
@Test
public void testGetNextRow() throws Exception {
  addRows(this.memstore);
  // Add more versions to make it a little more interesting.
  Thread.sleep(1);
  addRows(this.memstore);
  Cell closestToEmpty = ((DefaultMemStore) this.memstore).getNextRow(KeyValue.LOWESTKEY);
  assertTrue(CellComparatorImpl.COMPARATOR.compareRows(closestToEmpty,
      new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
  for (int i = 0; i < ROW_COUNT; i++) {
    Cell nr = ((DefaultMemStore) this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i),
        System.currentTimeMillis()));
    if (i + 1 == ROW_COUNT) {
      assertNull(nr);
    } else {
      assertTrue(CellComparatorImpl.COMPARATOR.compareRows(nr,
          new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
    }
  }
  //starting from each row, validate results should contain the starting row
  Configuration conf = HBaseConfiguration.create();
  for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
    ScanInfo scanInfo =
        new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE, KeepDeletedCells.FALSE,
            HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false);
    try (InternalScanner scanner =
        new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null,
            memstore.getScanners(0))) {
      List<Cell> results = new ArrayList<>();
      for (int i = 0; scanner.next(results); i++) {
        int rowId = startRowId + i;
        Cell left = results.get(0);
        byte[] row1 = Bytes.toBytes(rowId);
        assertTrue("Row name",
          CellComparatorImpl.COMPARATOR.compareRows(left, row1, 0, row1.length) == 0);
        assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
        List<Cell> row = new ArrayList<>();
        for (Cell kv : results) {
          row.add(kv);
        }
        isExpectedRowWithoutTimestamps(rowId, row);
        // Clear out set. Otherwise row results accumulate.
        results.clear();
      }
    }
  }
}
 
Example 11
Source Project: hbase   File: TestCompactingMemStore.java    License: Apache License 2.0 4 votes vote down vote up
/** Test getNextRow from memstore
 * @throws InterruptedException
 */
@Override
@Test
public void testGetNextRow() throws Exception {
  addRows(this.memstore);
  // Add more versions to make it a little more interesting.
  Thread.sleep(1);
  addRows(this.memstore);
  Cell closestToEmpty = ((CompactingMemStore)this.memstore).getNextRow(KeyValue.LOWESTKEY);
  assertTrue(CellComparator.getInstance().compareRows(closestToEmpty,
      new KeyValue(Bytes.toBytes(0), System.currentTimeMillis())) == 0);
  for (int i = 0; i < ROW_COUNT; i++) {
    Cell nr = ((CompactingMemStore)this.memstore).getNextRow(new KeyValue(Bytes.toBytes(i),
        System.currentTimeMillis()));
    if (i + 1 == ROW_COUNT) {
      assertNull(nr);
    } else {
      assertTrue(CellComparator.getInstance().compareRows(nr,
          new KeyValue(Bytes.toBytes(i + 1), System.currentTimeMillis())) == 0);
    }
  }
  //starting from each row, validate results should contain the starting row
  Configuration conf = HBaseConfiguration.create();
  for (int startRowId = 0; startRowId < ROW_COUNT; startRowId++) {
    ScanInfo scanInfo = new ScanInfo(conf, FAMILY, 0, 1, Integer.MAX_VALUE,
        KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, this.memstore.getComparator(), false);
    try (InternalScanner scanner =
        new StoreScanner(new Scan().withStartRow(Bytes.toBytes(startRowId)), scanInfo, null,
            memstore.getScanners(0))) {
      List<Cell> results = new ArrayList<>();
      for (int i = 0; scanner.next(results); i++) {
        int rowId = startRowId + i;
        Cell left = results.get(0);
        byte[] row1 = Bytes.toBytes(rowId);
        assertTrue("Row name",
          CellComparator.getInstance().compareRows(left, row1, 0, row1.length) == 0);
        assertEquals("Count of columns", QUALIFIER_COUNT, results.size());
        List<Cell> row = new ArrayList<>();
        for (Cell kv : results) {
          row.add(kv);
        }
        isExpectedRowWithoutTimestamps(rowId, row);
        // Clear out set. Otherwise row results accumulate.
        results.clear();
      }
    }
  }
}
 
Example 12
Source Project: hbase   File: TestStoreScanner.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testDeleteMarkerLongevity() throws Exception {
  try {
    final long now = System.currentTimeMillis();
    EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
      @Override
      public long currentTime() {
        return now;
      }
    });
    KeyValue[] kvs = new KeyValue[]{
      /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 100, KeyValue.Type.DeleteFamily), // live
      /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 1000, KeyValue.Type.DeleteFamily), // expired
      /*2*/ create("R1", "cf", "a", now - 50,
      KeyValue.Type.Put, "v3"), // live
      /*3*/ create("R1", "cf", "a", now - 55,
      KeyValue.Type.Delete, "dontcare"), // live
      /*4*/ create("R1", "cf", "a", now - 55,
      KeyValue.Type.Put, "deleted-version v2"), // deleted
      /*5*/ create("R1", "cf", "a", now - 60,
      KeyValue.Type.Put, "v1"), // live
      /*6*/ create("R1", "cf", "a", now - 65,
      KeyValue.Type.Put, "v0"), // max-version reached
      /*7*/ create("R1", "cf", "a",
      now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
      /*8*/ create("R1", "cf", "b", now - 600,
      KeyValue.Type.DeleteColumn, "dont-care"), //expired
      /*9*/ create("R1", "cf", "b", now - 70,
      KeyValue.Type.Put, "v2"), //live
      /*10*/ create("R1", "cf", "b", now - 750,
      KeyValue.Type.Put, "v1"), //expired
      /*11*/ create("R1", "cf", "c", now - 500,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*12*/ create("R1", "cf", "c", now - 600,
      KeyValue.Type.Put, "v1"), //expired
      /*13*/ create("R1", "cf", "c", now - 1000,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*14*/ create("R1", "cf", "d", now - 60,
      KeyValue.Type.Put, "expired put"), //live
      /*15*/ create("R1", "cf", "d", now - 100,
      KeyValue.Type.Delete, "not-expired delete"), //live
    };
    List<KeyValueScanner> scanners = scanFixture(kvs);
    ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
      0 /* minVersions */,
      2 /* maxVersions */, 500 /* ttl */,
      KeepDeletedCells.FALSE /* keepDeletedCells */,
      HConstants.DEFAULT_BLOCKSIZE /* block size */,
      200, /* timeToPurgeDeletes */
      CellComparator.getInstance(), false);
    try (StoreScanner scanner =
        new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
      List<Cell> results = new ArrayList<>();
      results = new ArrayList<>();
      assertEquals(true, scanner.next(results));
      assertEquals(kvs[0], results.get(0));
      assertEquals(kvs[2], results.get(1));
      assertEquals(kvs[3], results.get(2));
      assertEquals(kvs[5], results.get(3));
      assertEquals(kvs[9], results.get(4));
      assertEquals(kvs[14], results.get(5));
      assertEquals(kvs[15], results.get(6));
      assertEquals(7, results.size());
    }
  } finally {
    EnvironmentEdgeManagerTestHelper.reset();
  }
}
 
Example 13
Source Project: hbase   File: TestReversibleScanners.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReversibleStoreScanner() throws IOException {
  // write data to one memstore and two store files
  FileSystem fs = TEST_UTIL.getTestFileSystem();
  Path hfilePath = new Path(new Path(
      TEST_UTIL.getDataTestDir("testReversibleStoreScanner"), "regionname"),
      "familyname");
  CacheConfig cacheConf = new CacheConfig(TEST_UTIL.getConfiguration());
  HFileContextBuilder hcBuilder = new HFileContextBuilder();
  hcBuilder.withBlockSize(2 * 1024);
  HFileContext hFileContext = hcBuilder.build();
  StoreFileWriter writer1 = new StoreFileWriter.Builder(
      TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
      hfilePath).withFileContext(hFileContext).build();
  StoreFileWriter writer2 = new StoreFileWriter.Builder(
      TEST_UTIL.getConfiguration(), cacheConf, fs).withOutputDir(
      hfilePath).withFileContext(hFileContext).build();

  MemStore memstore = new DefaultMemStore();
  writeMemstoreAndStoreFiles(memstore, new StoreFileWriter[] { writer1,
      writer2 });

  HStoreFile sf1 = new HStoreFile(fs, writer1.getPath(), TEST_UTIL.getConfiguration(), cacheConf,
      BloomType.NONE, true);

  HStoreFile sf2 = new HStoreFile(fs, writer2.getPath(), TEST_UTIL.getConfiguration(), cacheConf,
      BloomType.NONE, true);

  ScanInfo scanInfo =
      new ScanInfo(TEST_UTIL.getConfiguration(), FAMILYNAME, 0, Integer.MAX_VALUE, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0, CellComparatorImpl.COMPARATOR, false);

  // Case 1.Test a full reversed scan
  Scan scan = new Scan();
  scan.setReversed(true);
  StoreScanner storeScanner =
      getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, QUALSIZE * ROWSIZE, ROWSIZE, false);

  // Case 2.Test reversed scan with a specified start row
  int startRowNum = ROWSIZE / 2;
  byte[] startRow = ROWS[startRowNum];
  scan.withStartRow(startRow);
  storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, QUALSIZE * (startRowNum + 1),
      startRowNum + 1, false);

  // Case 3.Test reversed scan with a specified start row and specified
  // qualifiers
  assertTrue(QUALSIZE > 2);
  scan.addColumn(FAMILYNAME, QUALS[0]);
  scan.addColumn(FAMILYNAME, QUALS[2]);
  storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, MAXMVCC);
  verifyCountAndOrder(storeScanner, 2 * (startRowNum + 1), startRowNum + 1,
      false);

  // Case 4.Test reversed scan with mvcc based on case 3
  for (int readPoint = 0; readPoint < MAXMVCC; readPoint++) {
    LOG.info("Setting read point to " + readPoint);
    storeScanner = getReversibleStoreScanner(memstore, sf1, sf2, scan, scanInfo, readPoint);
    int expectedRowCount = 0;
    int expectedKVCount = 0;
    for (int i = startRowNum; i >= 0; i--) {
      int kvCount = 0;
      if (makeMVCC(i, 0) <= readPoint) {
        kvCount++;
      }
      if (makeMVCC(i, 2) <= readPoint) {
        kvCount++;
      }
      if (kvCount > 0) {
        expectedRowCount++;
        expectedKVCount += kvCount;
      }
    }
    verifyCountAndOrder(storeScanner, expectedKVCount, expectedRowCount,
        false);
  }
}