Java Code Examples for org.apache.hadoop.hbase.client.Scan#setMaxVersions()
The following examples show how to use
org.apache.hadoop.hbase.client.Scan#setMaxVersions() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 6 votes |
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionInfo().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
Example 2
Source File: TestUtil.java From phoenix with Apache License 2.0 | 6 votes |
public static int getRowCount(Table table, boolean isRaw) throws IOException { Scan s = new Scan(); s.setRaw(isRaw);; s.setMaxVersions(); int rows = 0; try (ResultScanner scanner = table.getScanner(s)) { Result result = null; while ((result = scanner.next()) != null) { rows++; CellScanner cellScanner = result.cellScanner(); Cell current = null; while (cellScanner.advance()) { current = cellScanner.current(); } } } return rows; }
Example 3
Source File: IndexTestingUtils.java From phoenix with Apache License 2.0 | 6 votes |
/** * Verify the state of the index table between the given key and time ranges against the list of * expected keyvalues. * @throws IOException */ @SuppressWarnings("javadoc") public static void verifyIndexTableAtTimestamp(Table index1, List<KeyValue> expected, long start, long end, byte[] startKey, byte[] endKey) throws IOException { LOGGER.debug("Scanning " + index1.getName().getNameAsString() + " between times (" + start + ", " + end + "] and keys: [" + Bytes.toString(startKey) + ", " + Bytes.toString(endKey) + "]."); Scan s = new Scan(startKey, endKey); // s.setRaw(true); s.setMaxVersions(); s.setTimeRange(start, end); List<Cell> received = new ArrayList<Cell>(); ResultScanner scanner = index1.getScanner(s); for (Result r : scanner) { received.addAll(r.listCells()); LOGGER.debug("Received: " + r.listCells()); } scanner.close(); assertEquals("Didn't get the expected kvs from the index table!", expected, received); }
Example 4
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 6 votes |
protected InternalScanner createStoreScanner(RegionCoprocessorEnvironment env, String action, TransactionVisibilityState snapshot, Store store, List<? extends KeyValueScanner> scanners, ScanType type, long earliestPutTs) throws IOException { if (snapshot == null) { if (LOG.isDebugEnabled()) { LOG.debug("Region " + env.getRegion().getRegionNameAsString() + ", no current transaction state found, defaulting to normal " + action + " scanner"); } return null; } // construct a dummy transaction from the latest snapshot Transaction dummyTx = TxUtils.createDummyTransaction(snapshot); Scan scan = new Scan(); // need to see all versions, since we filter out excludes and applications may rely on multiple versions scan.setMaxVersions(); scan.setFilter( new IncludeInProgressFilter(dummyTx.getVisibilityUpperBound(), snapshot.getInvalid(), getTransactionFilter(dummyTx, type, null))); return new StoreScanner(store, store.getScanInfo(), scan, scanners, type, store.getSmallestReadPoint(), earliestPutTs); }
Example 5
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { Transaction tx = getFromOperation(scan); if (tx != null) { projectFamilyDeletes(scan); scan.setMaxVersions(); scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData), TxUtils.getMaxVisibleTimestamp(tx)); Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter()); scan.setFilter(newFilter); } return s; }
Example 6
Source File: HBaseAccessor.java From pxf with Apache License 2.0 | 5 votes |
/** * Creates the Scan object used to describe the query * requested from HBase. * As the row key column always gets returned, no need to ask for it. */ private void createScanner() throws Exception { scanDetails = new Scan(); // Return only one version (latest) scanDetails.setMaxVersions(1); addColumns(); addFilters(); }
Example 7
Source File: TransactionProcessor.java From phoenix-tephra with Apache License 2.0 | 5 votes |
@Override public RegionScanner preScannerOpen(ObserverContext<RegionCoprocessorEnvironment> e, Scan scan, RegionScanner s) throws IOException { Transaction tx = getFromOperation(scan); if (tx != null) { projectFamilyDeletes(scan); scan.setMaxVersions(); scan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttlByFamily, tx, readNonTxnData), TxUtils.getMaxVisibleTimestamp(tx)); Filter newFilter = getTransactionFilter(tx, ScanType.USER_SCAN, scan.getFilter()); scan.setFilter(newFilter); } return s; }
Example 8
Source File: DropIndexTest.java From hgraphdb with Apache License 2.0 | 5 votes |
private void verifyTableCount(final Table table, final int count) throws IOException { Scan scan = new Scan(); scan.setMaxVersions(1); ResultScanner scanner = table.getScanner(scan); int i = 0; for (Result r : scanner) { i++; } assertEquals(count, i); scanner.close(); }
Example 9
Source File: SpliceDefaultCompactor.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
protected InternalScanner createScanner(HStore store, List<StoreFileScanner> scanners, ScanType scanType, long smallestReadPoint, long earliestPutTs) throws IOException { if (LOG.isTraceEnabled()) SpliceLogUtils.trace(LOG,"createScanner"); Scan scan = new Scan(); scan.setMaxVersions(store.getColumnFamilyDescriptor().getMaxVersions()); return new StoreScanner(store, store.getScanInfo(), scanners, scanType, smallestReadPoint, earliestPutTs); }
Example 10
Source File: WALReplayWithIndexWritesAndCompressedWALIT.java From phoenix with Apache License 2.0 | 5 votes |
@SuppressWarnings("deprecation") private int getKeyValueCount(Table table) throws IOException { Scan scan = new Scan(); scan.setMaxVersions(Integer.MAX_VALUE - 1); ResultScanner results = table.getScanner(scan); int count = 0; for (Result res : results) { count += res.listCells().size(); LOGGER.debug(count + ") " + res); } results.close(); return count; }
Example 11
Source File: IndexManagementUtil.java From phoenix with Apache License 2.0 | 5 votes |
public static Scan newLocalStateScan(List<? extends Iterable<? extends ColumnReference>> refsArray) { Scan s = new Scan(); s.setRaw(true); // add the necessary columns to the scan for (Iterable<? extends ColumnReference> refs : refsArray) { for (ColumnReference ref : refs) { s.addFamily(ref.getFamily()); } } s.setMaxVersions(); return s; }
Example 12
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testDeleteFiltering() throws Exception { String tableName = "TestDeleteFiltering"; byte[] familyBytes = Bytes.toBytes("f"); byte[] columnBytes = Bytes.toBytes("c"); HRegion region = createRegion(tableName, familyBytes, 0); try { region.initialize(); TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get(); LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache)); byte[] row = Bytes.toBytes(1); for (int i = 4; i < V.length; i++) { Put p = new Put(row); p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i])); region.put(p); } // delete from the third entry back // take that cell's timestamp + 1 to simulate a delete in a new tx long deleteTs = V[5] + 1; Delete d = new Delete(row, deleteTs); LOG.info("Issuing delete at timestamp " + deleteTs); // row deletes are not yet supported (TransactionAwareHTable normally handles this) d.deleteColumns(familyBytes, columnBytes); region.delete(d); List<Cell> results = Lists.newArrayList(); // force a flush to clear the data // during flush, we should drop the deleted version, but not the others LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString()); region.flushcache(true, false); // now a normal scan should return row with versions at: V[8], V[6]. // V[7] is invalid and V[5] and prior are deleted. Scan scan = new Scan(); scan.setMaxVersions(10); RegionScanner regionScanner = region.getScanner(scan); // should be only one row assertFalse(regionScanner.next(results)); assertKeyValueMatches(results, 1, new long[]{V[8], V[6], deleteTs}, new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]}); } finally { region.close(); } }
Example 13
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testPreExistingData() throws Exception { String tableName = "TestPreExistingData"; byte[] familyBytes = Bytes.toBytes("f"); long ttlMillis = TimeUnit.DAYS.toMillis(14); HRegion region = createRegion(tableName, familyBytes, ttlMillis); try { region.initialize(); // timestamps for pre-existing, non-transactional data long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS; long older = now - ttlMillis / 2; long newer = now - ttlMillis / 3; // timestamps for transactional data long nowTx = txVisibilityState.getVisibilityUpperBound(); long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS; long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS; Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); ttls.put(familyBytes, ttlMillis); List<Cell> cells = new ArrayList<>(); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11"))); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32"))); // Write non-transactional and transactional data for (Cell c : cells) { region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue())); } Scan rawScan = new Scan(); rawScan.setMaxVersions(); Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState); Scan txScan = new Scan(); txScan.setMaxVersions(); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // read all back with raw scanner scanAndAssert(region, cells, rawScan); // read all back with transaction filter scanAndAssert(region, cells, txScan); // force a flush to clear the memstore region.flushcache(); scanAndAssert(region, cells, txScan); // force a major compaction to remove any expired cells region.compactStores(true); scanAndAssert(region, cells, txScan); // Reduce TTL, this should make cells with timestamps older and olderTx expire long newTtl = ttlMillis / 2 - 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // Raw scan should still give all cells scanAndAssert(region, cells, rawScan); // However, tx scan should not return expired cells scanAndAssert(region, select(cells, 1, 3, 5), txScan); region.flushcache(); scanAndAssert(region, cells, rawScan); // force a major compaction to remove any expired cells region.compactStores(true); // This time raw scan too should not return expired cells, as they would be dropped during major compaction scanAndAssert(region, select(cells, 1, 3, 5), rawScan); // Reduce TTL again to 1 ms, this should expire all cells newTtl = 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // force a major compaction to remove expired cells region.compactStores(true); // This time raw scan should not return any cells, as all cells have expired. scanAndAssert(region, Collections.<Cell>emptyList(), rawScan); } finally { region.close(); } }
Example 14
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testPreExistingData() throws Exception { String tableName = "TestPreExistingData"; byte[] familyBytes = Bytes.toBytes("f"); long ttlMillis = TimeUnit.DAYS.toMillis(14); HRegion region = createRegion(tableName, familyBytes, ttlMillis); try { region.initialize(); // timestamps for pre-existing, non-transactional data long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS; long older = now - ttlMillis / 2; long newer = now - ttlMillis / 3; // timestamps for transactional data long nowTx = txVisibilityState.getVisibilityUpperBound(); long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS; long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS; Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); ttls.put(familyBytes, ttlMillis); List<Cell> cells = new ArrayList<>(); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11"))); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32"))); // Write non-transactional and transactional data for (Cell c : cells) { region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue())); } Scan rawScan = new Scan(); rawScan.setMaxVersions(); Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState); Scan txScan = new Scan(); txScan.setMaxVersions(); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // read all back with raw scanner scanAndAssert(region, cells, rawScan); // read all back with transaction filter scanAndAssert(region, cells, txScan); // force a flush to clear the memstore region.flushcache(); scanAndAssert(region, cells, txScan); // force a major compaction to remove any expired cells region.compactStores(true); scanAndAssert(region, cells, txScan); // Reduce TTL, this should make cells with timestamps older and olderTx expire long newTtl = ttlMillis / 2 - 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // Raw scan should still give all cells scanAndAssert(region, cells, rawScan); // However, tx scan should not return expired cells scanAndAssert(region, select(cells, 1, 3, 5), txScan); region.flushcache(); scanAndAssert(region, cells, rawScan); // force a major compaction to remove any expired cells region.compactStores(true); // This time raw scan too should not return expired cells, as they would be dropped during major compaction scanAndAssert(region, select(cells, 1, 3, 5), rawScan); // Reduce TTL again to 1 ms, this should expire all cells newTtl = 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // force a major compaction to remove expired cells region.compactStores(true); // This time raw scan should not return any cells, as all cells have expired. scanAndAssert(region, Collections.<Cell>emptyList(), rawScan); } finally { region.close(); } }
Example 15
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testPreExistingData() throws Exception { String tableName = "TestPreExistingData"; byte[] familyBytes = Bytes.toBytes("f"); long ttlMillis = TimeUnit.DAYS.toMillis(14); HRegion region = createRegion(tableName, familyBytes, ttlMillis); try { region.initialize(); // timestamps for pre-existing, non-transactional data long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS; long older = now - ttlMillis / 2; long newer = now - ttlMillis / 3; // timestamps for transactional data long nowTx = txVisibilityState.getVisibilityUpperBound(); long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS; long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS; Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); ttls.put(familyBytes, ttlMillis); List<Cell> cells = new ArrayList<>(); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11"))); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32"))); // Write non-transactional and transactional data for (Cell c : cells) { region.put(new Put(c.getRow()).add(c.getFamily(), c.getQualifier(), c.getTimestamp(), c.getValue())); } Scan rawScan = new Scan(); rawScan.setMaxVersions(); Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState); Scan txScan = new Scan(); txScan.setMaxVersions(); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // read all back with raw scanner scanAndAssert(region, cells, rawScan); // read all back with transaction filter scanAndAssert(region, cells, txScan); // force a flush to clear the memstore region.flushcache(true, false); scanAndAssert(region, cells, txScan); // force a major compaction to remove any expired cells region.compact(true); scanAndAssert(region, cells, txScan); // Reduce TTL, this should make cells with timestamps older and olderTx expire long newTtl = ttlMillis / 2 - 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // Raw scan should still give all cells scanAndAssert(region, cells, rawScan); // However, tx scan should not return expired cells scanAndAssert(region, select(cells, 1, 3, 5), txScan); region.flushcache(true, false); scanAndAssert(region, cells, rawScan); // force a major compaction to remove any expired cells region.compact(true); // This time raw scan too should not return expired cells, as they would be dropped during major compaction scanAndAssert(region, select(cells, 1, 3, 5), rawScan); // Reduce TTL again to 1 ms, this should expire all cells newTtl = 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // force a major compaction to remove expired cells region.compact(true); // This time raw scan should not return any cells, as all cells have expired. scanAndAssert(region, Collections.<Cell>emptyList(), rawScan); } finally { region.close(); } }
Example 16
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testPreExistingData() throws Exception { String tableName = "TestPreExistingData"; byte[] familyBytes = Bytes.toBytes("f"); long ttlMillis = TimeUnit.DAYS.toMillis(14); HRegion region = createRegion(tableName, familyBytes, ttlMillis); try { region.initialize(); // timestamps for pre-existing, non-transactional data long now = txVisibilityState.getVisibilityUpperBound() / TxConstants.MAX_TX_PER_MS; long older = now - ttlMillis / 2; long newer = now - ttlMillis / 3; // timestamps for transactional data long nowTx = txVisibilityState.getVisibilityUpperBound(); long olderTx = nowTx - (ttlMillis / 2) * TxConstants.MAX_TX_PER_MS; long newerTx = nowTx - (ttlMillis / 3) * TxConstants.MAX_TX_PER_MS; Map<byte[], Long> ttls = Maps.newTreeMap(Bytes.BYTES_COMPARATOR); ttls.put(familyBytes, ttlMillis); List<Cell> cells = new ArrayList<>(); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v11"))); cells.add(new KeyValue(Bytes.toBytes("r1"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v12"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c1"), older, Bytes.toBytes("v21"))); cells.add(new KeyValue(Bytes.toBytes("r2"), familyBytes, Bytes.toBytes("c2"), newer, Bytes.toBytes("v22"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c1"), olderTx, Bytes.toBytes("v31"))); cells.add(new KeyValue(Bytes.toBytes("r3"), familyBytes, Bytes.toBytes("c2"), newerTx, Bytes.toBytes("v32"))); // Write non-transactional and transactional data for (Cell c : cells) { region.put(new Put(CellUtil.cloneRow(c)).addColumn(CellUtil.cloneFamily(c), CellUtil.cloneQualifier(c), c.getTimestamp(), CellUtil.cloneValue(c))); } Scan rawScan = new Scan(); rawScan.setMaxVersions(); Transaction dummyTransaction = TxUtils.createDummyTransaction(txVisibilityState); Scan txScan = new Scan(); txScan.setMaxVersions(); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // read all back with raw scanner scanAndAssert(region, cells, rawScan); // read all back with transaction filter scanAndAssert(region, cells, txScan); // force a flush to clear the memstore region.flushcache(true, false, new FlushLifeCycleTracker() { }); scanAndAssert(region, cells, txScan); // force a major compaction to remove any expired cells region.compact(true); scanAndAssert(region, cells, txScan); // Reduce TTL, this should make cells with timestamps older and olderTx expire long newTtl = ttlMillis / 2 - 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // Raw scan should still give all cells scanAndAssert(region, cells, rawScan); // However, tx scan should not return expired cells scanAndAssert(region, select(cells, 1, 3, 5), txScan); region.flushcache(true, false, new FlushLifeCycleTracker() { }); scanAndAssert(region, cells, rawScan); // force a major compaction to remove any expired cells region.compact(true); // This time raw scan too should not return expired cells, as they would be dropped during major compaction scanAndAssert(region, select(cells, 1, 3, 5), rawScan); // Reduce TTL again to 1 ms, this should expire all cells newTtl = 1; region = updateTtl(region, familyBytes, newTtl); ttls.put(familyBytes, newTtl); txScan.setTimeRange(TxUtils.getOldestVisibleTimestamp(ttls, dummyTransaction, true), TxUtils.getMaxVisibleTimestamp(dummyTransaction)); txScan.setFilter(TransactionFilters.getVisibilityFilter(dummyTransaction, ttls, false, ScanType.USER_SCAN)); // force a major compaction to remove expired cells region.compact(true); // This time raw scan should not return any cells, as all cells have expired. scanAndAssert(region, Collections.<Cell>emptyList(), rawScan); } finally { region.close(); } }
Example 17
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testDataJanitorRegionScanner() throws Exception { String tableName = "TestRegionScanner"; byte[] familyBytes = Bytes.toBytes("f"); byte[] columnBytes = Bytes.toBytes("c"); HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3)); try { region.initialize(); TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get(); LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache)); for (int i = 1; i <= 8; i++) { for (int k = 1; k <= i; k++) { Put p = new Put(Bytes.toBytes(i)); p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k])); region.put(p); } } List<Cell> results = Lists.newArrayList(); // force a flush to clear the data // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString()); Region.FlushResult flushResult = region.flushcache(true, false); Assert.assertTrue("Unexpected flush result: " + flushResult, flushResult.isFlushSucceeded()); // now a normal scan should only return the valid rows // do not use a filter here to test that cleanup works on flush Scan scan = new Scan(); scan.setMaxVersions(10); RegionScanner regionScanner = region.getScanner(scan); // first returned value should be "4" with version "4" results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 4, new long[]{V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 5, new long[] {V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 6, new long[]{V[6], V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 7, new long[]{V[6], V[4]}); results.clear(); assertFalse(regionScanner.next(results)); assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]}); } finally { region.close(); } }
Example 18
Source File: ProcessRecordService.java From hraven with Apache License 2.0 | 4 votes |
/** * @param cluster for which to return the last ProcessRecord. * @param compareOp to apply to the processState argument. If * {@link CompareOp#NO_OP} is passed, then no filter is used at all, * and processState argument is ignored. * @param processState return rows where the compareOp applies. * @param maxCount the maximum number of results to return. * @param processFileSubstring return rows where the process file path * contains this string. If <code>null</code> or empty string, then * no filtering is applied. * @return the last process record that is not in {@link ProcessState#CREATED} * state. Note that no records with a maxModificationTime of 0 * (beginning of time) will be returned * @throws IOException */ public List<ProcessRecord> getProcessRecords(String cluster, CompareOp compareOp, ProcessState processState, int maxCount, String processFileSubstring) throws IOException { Scan scan = new Scan(); // Pull data only for our cluster scan.setStartRow( keyConv.toBytes(new ProcessRecordKey(cluster, Long.MAX_VALUE))); // Records are sorted in reverse order, so the last one for this cluster // would be the one with a modification time at the beginning of time. scan.setStopRow(keyConv.toBytes(new ProcessRecordKey(cluster, 0))); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.MIN_MOD_TIME_MILLIS_COLUMN_BYTES); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.PROCESSED_JOB_FILES_COLUMN_BYTES); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.PROCESS_FILE_COLUMN_BYTES); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.PROCESSING_STATE_COLUMN_BYTES); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.MIN_JOB_ID_COLUMN_BYTES); scan.addColumn(Constants.INFO_FAM_BYTES, Constants.MAX_JOB_ID_COLUMN_BYTES); scan.setMaxVersions(1); FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL); // Filter on process state only when needed. if (!NO_OP.equals(compareOp)) { byte[] filterColumnValue = Bytes.toBytes(processState.getCode()); Filter processingStatefilter = new SingleColumnValueFilter( Constants.INFO_FAM_BYTES, Constants.PROCESSING_STATE_COLUMN_BYTES, compareOp, filterColumnValue); filterList.addFilter(processingStatefilter); } // Filter on process file only when needed if (processFileSubstring != null && processFileSubstring.length() > 0) { SubstringComparator ssc = new SubstringComparator(processFileSubstring); Filter processFileFilter = new SingleColumnValueFilter(Constants.INFO_FAM_BYTES, Constants.PROCESS_FILE_COLUMN_BYTES, EQUAL, ssc); filterList.addFilter(processFileFilter); } // Add filters only if any filter was actually needed. if (filterList.getFilters().size() > 0) { scan.setFilter(filterList); } ResultScanner scanner = null; List<ProcessRecord> records = null; Table processRecordTable = null; try { processRecordTable = hbaseConnection .getTable(TableName.valueOf(Constants.JOB_FILE_PROCESS_TABLE)); scanner = processRecordTable.getScanner(scan); records = createFromResults(scanner, maxCount); } finally { if (scanner != null) { scanner.close(); } if (processRecordTable != null) { processRecordTable.close(); } } return records; }
Example 19
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testDeleteFiltering() throws Exception { String tableName = "TestDeleteFiltering"; byte[] familyBytes = Bytes.toBytes("f"); byte[] columnBytes = Bytes.toBytes("c"); HRegion region = createRegion(tableName, familyBytes, 0); try { region.initialize(); TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get(); LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache)); byte[] row = Bytes.toBytes(1); for (int i = 4; i < V.length; i++) { Put p = new Put(row); p.add(familyBytes, columnBytes, V[i], Bytes.toBytes(V[i])); region.put(p); } // delete from the third entry back // take that cell's timestamp + 1 to simulate a delete in a new tx long deleteTs = V[5] + 1; Delete d = new Delete(row, deleteTs); LOG.info("Issuing delete at timestamp " + deleteTs); // row deletes are not yet supported (TransactionAwareHTable normally handles this) d.deleteColumns(familyBytes, columnBytes); region.delete(d); List<Cell> results = Lists.newArrayList(); // force a flush to clear the data // during flush, we should drop the deleted version, but not the others LOG.info("Flushing region " + region.getRegionInfo().getRegionNameAsString()); region.flushcache(true, false); // now a normal scan should return row with versions at: V[8], V[6]. // V[7] is invalid and V[5] and prior are deleted. Scan scan = new Scan(); scan.setMaxVersions(10); RegionScanner regionScanner = region.getScanner(scan); // should be only one row assertFalse(regionScanner.next(results)); assertKeyValueMatches(results, 1, new long[]{V[8], V[6], deleteTs}, new byte[][]{Bytes.toBytes(V[8]), Bytes.toBytes(V[6]), new byte[0]}); } finally { region.close(); } }
Example 20
Source File: TransactionProcessorTest.java From phoenix-tephra with Apache License 2.0 | 4 votes |
@Test public void testDataJanitorRegionScanner() throws Exception { String tableName = "TestRegionScanner"; byte[] familyBytes = Bytes.toBytes("f"); byte[] columnBytes = Bytes.toBytes("c"); HRegion region = createRegion(tableName, familyBytes, TimeUnit.HOURS.toMillis(3)); try { region.initialize(); TransactionStateCache cache = new TransactionStateCacheSupplier(conf).get(); LOG.info("Coprocessor is using transaction state: " + waitForTransactionState(cache)); for (int i = 1; i <= 8; i++) { for (int k = 1; k <= i; k++) { Put p = new Put(Bytes.toBytes(i)); p.add(familyBytes, columnBytes, V[k], Bytes.toBytes(V[k])); region.put(p); } } List<Cell> results = Lists.newArrayList(); // force a flush to clear the data // during flush, the coprocessor should drop all KeyValues with timestamps in the invalid set LOG.info("Flushing region " + region.getRegionNameAsString()); HRegion.FlushResult flushResult = region.flushcache(); Assert.assertTrue("Unexpected flush result: " + flushResult.toString(), flushResult.isFlushSucceeded()); // now a normal scan should only return the valid rows // do not use a filter here to test that cleanup works on flush Scan scan = new Scan(); scan.setMaxVersions(10); RegionScanner regionScanner = region.getScanner(scan); // first returned value should be "4" with version "4" results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 4, new long[] {V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 5, new long[] {V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 6, new long[] {V[6], V[4]}); results.clear(); assertTrue(regionScanner.next(results)); assertKeyValueMatches(results, 7, new long[] {V[6], V[4]}); results.clear(); assertFalse(regionScanner.next(results)); assertKeyValueMatches(results, 8, new long[] {V[8], V[6], V[4]}); } finally { region.close(); } }