Java Code Examples for org.apache.hadoop.hbase.KeyValue.Type

The following examples show how to use org.apache.hadoop.hbase.KeyValue.Type. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix   Source File: SchemaUtil.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private static KeyValue upgradeTo3(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    int newLength = keyValue.getRowLength() + 1;
    byte[] newKey = new byte[newLength];
    newKey[0] = QueryConstants.SEPARATOR_BYTE;
    System.arraycopy(buf, keyValue.getRowOffset(), newKey, 1, keyValue.getRowLength());
    byte[] valueBuf = updateValueIfNecessary(keyValue);
    int valueOffset = keyValue.getValueOffset();
    int valueLength = keyValue.getValueLength();
    if (valueBuf != buf) {
        valueOffset = 0;
        valueLength = valueBuf.length;
    }
    return new KeyValue(newKey, 0, newLength,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), Type.codeToType(keyValue.getType()),
            valueBuf, valueOffset, valueLength);
}
 
Example 2
private void validate(KeyValue kv, byte[] row, byte[] family, byte[] qualifier, long ts,
    Type type, byte[] value) throws IOException {
  DataOutputBuffer out = new DataOutputBuffer();
  kv.write(out);
  out.close();
  byte[] data = out.getData();
  // read it back in
  KeyValue read = new KeyValue();
  DataInputBuffer in = new DataInputBuffer();
  in.reset(data, data.length);
  read.readFields(in);
  in.close();

  // validate that its the same
  assertTrue("Row didn't match!", Bytes.equals(row, read.getRow()));
  assertTrue("Family didn't match!", Bytes.equals(family, read.getFamily()));
  assertTrue("Qualifier didn't match!", Bytes.equals(qualifier, read.getQualifier()));
  assertTrue("Value didn't match!", Bytes.equals(value, read.getValue()));
  assertEquals("Timestamp didn't match", ts, read.getTimestamp());
  assertEquals("Type didn't match", type.getCode(), read.getType());
}
 
Example 3
Source Project: phoenix   Source File: IndexMaintainer.java    License: Apache License 2.0 6 votes vote down vote up
public boolean isRowDeleted(Collection<KeyValue> pendingUpdates) {
    int nDeleteCF = 0;
    for (KeyValue kv : pendingUpdates) {
        if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()) {
            nDeleteCF++;
            boolean isEmptyCF = Bytes.compareTo(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), 
              dataEmptyKeyValueCF, 0, dataEmptyKeyValueCF.length) == 0;
            // This is what a delete looks like on the client side for immutable indexing...
            if (isEmptyCF) {
                return true;
            }
        }
    }
    // This is what a delete looks like on the server side for mutable indexing...
    return nDeleteCF == this.nDataCFs;
}
 
Example 4
Source Project: phoenix   Source File: ColumnProjectionFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void filterRowCells(List<Cell> kvs) throws IOException {
    if (kvs.isEmpty()) return;
    Cell firstKV = kvs.get(0);
    Iterator<Cell> itr = kvs.iterator();
    while (itr.hasNext()) {
        Cell kv = itr.next();
        ptr.set(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength());
        if (this.columnsTracker.containsKey(ptr)) {
            Set<ImmutableBytesPtr> cols = this.columnsTracker.get(ptr);
            ptr.set(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
            if (cols != null && !(cols.contains(ptr))) {
                itr.remove();
            }
        } else {
            itr.remove();
        }
    }
    // make sure we're not holding to any of the byte[]'s
    ptr.set(HConstants.EMPTY_BYTE_ARRAY);
    if (kvs.isEmpty()) {
        kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(),firstKV.getRowLength(), this.emptyCFName,
                0, this.emptyCFName.length, QueryConstants.EMPTY_COLUMN_BYTES, 0,
                QueryConstants.EMPTY_COLUMN_BYTES.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0));
    }
}
 
Example 5
Source Project: phoenix   Source File: TestIndexMemStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR);
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  KeyValueScanner scanner = store.getScanner();
  KeyValue first = KeyValue.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 
Example 6
Source Project: hbase-indexer   Source File: ColumnBasedIndexerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_UpdateAndDeleteCombinedForSameCell_UpdateFirst() throws IOException {
    KeyValue toAdd = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                  Bytes.toBytes("_qual_"),
                                  Bytes.toBytes("value"));
    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"),
                                     0L, Type.DeleteColumn);
    RowData addEventRowData = createEventRowData("_row_", toAdd);
    RowData deleteEventRowData = createEventRowData("_row_", toDelete);

    indexer.calculateIndexUpdates(ImmutableList.of(addEventRowData, deleteEventRowData), updateCollector);

    assertEquals(Lists.newArrayList("_row_-_cf_-_qual_"), updateCollector.getIdsToDelete());
    assertTrue(updateCollector.getDocumentsToAdd().isEmpty());
}
 
Example 7
Source Project: phoenix   Source File: TestApplyAndFilterDeletesFilter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * DeleteFamily markers should delete everything from that timestamp backwards, but not hide
 * anything forwards
 */
@Test
public void testDeleteFamilyCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue df = createKvForType(Type.DeleteFamily, 11);
  KeyValue put = createKvForType(Type.Put, 12);

  assertEquals("Didn't filter out delete family", ReturnCode.SKIP, filter.filterKeyValue(df));
  assertEquals("Filtered out put with newer TS than delete family", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // older kv shouldn't be visible
  put = createKvForType(Type.Put, 10);
  assertEquals("Didn't filter out older put, covered by DeleteFamily marker",
    ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(put));

  // next seek should be past the families
  assertEquals(KeyValue.LOWESTKEY, filter.getNextCellHint(put));
}
 
Example 8
Source Project: phoenix   Source File: TestApplyAndFilterDeletesFilter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Point deletes should only cover the exact entry they are tied to. Earlier puts should always
 * show up.
 */
@Test
public void testCoveringPointDelete() {
  // start with doing a family delete, so we will seek to the next column
  KeyValue kv = createKvForType(Type.Delete);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  filter.filterKeyValue(kv);
  KeyValue put = createKvForType(Type.Put);
  assertEquals("Didn't filter out put with same timestamp!", ReturnCode.SKIP,
    filter.filterKeyValue(put));
  // we should filter out the exact same put again, which could occur with the kvs all kept in the
  // same memstore
  assertEquals("Didn't filter out put with same timestamp on second call!", ReturnCode.SKIP,
    filter.filterKeyValue(put));

  // ensure then that we don't filter out a put with an earlier timestamp (though everything else
  // matches)
  put = createKvForType(Type.Put, ts - 1);
  assertEquals("Didn't accept put that has an earlier ts than the covering delete!",
    ReturnCode.INCLUDE, filter.filterKeyValue(put));
}
 
Example 9
/**
 * DeleteFamily markers should delete everything from that timestamp backwards, but not hide
 * anything forwards
 */
@Test
public void testDeleteFamilyCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue df = createKvForType(Type.DeleteFamily, 11);
  KeyValue put = createKvForType(Type.Put, 12);

  assertEquals("Didn't filter out delete family", ReturnCode.SKIP, filter.filterKeyValue(df));
  assertEquals("Filtered out put with newer TS than delete family", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // older kv shouldn't be visible
  put = createKvForType(Type.Put, 10);
  assertEquals("Didn't filter out older put, covered by DeleteFamily marker",
    ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(put));

  // next seek should be past the families
  assertEquals(KeyValue.LOWESTKEY, filter.getNextKeyHint(put));
}
 
Example 10
@Test
public void testFilterNotExistColumnFamilyEdits() {
  List<List<Entry>> entryList = new ArrayList<>();
  // should be filtered
  Cell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, System.currentTimeMillis(),
      Type.Put, VALUE);
  Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c1));
  entryList.add(Lists.newArrayList(e1));
  // should be kept
  Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, System.currentTimeMillis(), Type.Put, VALUE);
  Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c2));
  entryList.add(Lists.newArrayList(e2, e1));
  List<List<Entry>> filtered = endpoint.filterNotExistColumnFamilyEdits(entryList);
  assertEquals(1, filtered.size());
  assertEquals(1, filtered.get(0).get(0).getEdit().getCells().size());
  Cell cell = filtered.get(0).get(0).getEdit().getCells().get(0);
  assertTrue(CellUtil.matchingFamily(cell, FAMILY));
}
 
Example 11
Source Project: hbase   Source File: TestCompactionScanQueryMatcher.java    License: Apache License 2.0 6 votes vote down vote up
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example 12
/**
 * Test that we don't cover other columns when we have a delete column.
 */
@Test
public void testDeleteColumnCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue d = createKvForType(Type.DeleteColumn, 12);
  byte[] qual2 = Bytes.add(qualifier, Bytes.toBytes("-other"));
  KeyValue put =
      new KeyValue(row, family, qual2, 0, qual2.length, 11, Type.Put, value, 0,
          value.length);

  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // different column put should still be visible
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // set a delete family, but in the past
  d = createKvForType(Type.DeleteFamily, 10);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // add back in the original delete column
  d = createKvForType(Type.DeleteColumn, 11);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // onto a different family, so that must be visible too
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));
}
 
Example 13
/**
 * Copied from {@link TestKeyValue}
 * @throws Exception
 */
@Test
public void testColumnCompare() throws Exception {
  final byte [] a = Bytes.toBytes("aaa");
  byte [] family1 = Bytes.toBytes("abc");
  byte [] qualifier1 = Bytes.toBytes("def");
  byte [] family2 = Bytes.toBytes("abcd");
  byte [] qualifier2 = Bytes.toBytes("ef");

  KeyValue aaa = new ClientKeyValue(a, family1, qualifier1, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family2, qualifier2));
  assertTrue(aaa.matchingColumn(family1, qualifier1));
  aaa = new ClientKeyValue(a, family2, qualifier2, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family1, qualifier1));
  assertTrue(aaa.matchingColumn(family2,qualifier2));
  byte [] nullQualifier = new byte[0];
  aaa = new ClientKeyValue(a, family1, nullQualifier, 0L, Type.Put, a);
  assertTrue(aaa.matchingColumn(family1,null));
  assertFalse(aaa.matchingColumn(family2,qualifier2));
}
 
Example 14
Source Project: hbase-indexer   Source File: ColumnBasedIndexerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_DeleteFamily() throws IOException {

    final String ROW_FIELD = "_row_field_";
    final String FAMILY_FIELD = "_family_field_";

    doReturn(ROW_FIELD).when(indexerConf).getRowField();
    doReturn(FAMILY_FIELD).when(indexerConf).getColumnFamilyField();

    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"), 0L, Type.DeleteFamily);
    RowData rowData = createEventRowData("_row_", toDelete);

    indexer.calculateIndexUpdates(Lists.newArrayList(rowData), updateCollector);

    assertEquals(ImmutableList.of("(" + ROW_FIELD + ":_row_)AND(" + FAMILY_FIELD + ":_cf_)"),
            updateCollector.getDeleteQueries());
    assertTrue(updateCollector.getIdsToDelete().isEmpty());
    assertTrue(updateCollector.getDocumentsToAdd().isEmpty());
}
 
Example 15
Source Project: hbase-indexer   Source File: ColumnBasedIndexerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_UpdateAndDeleteCombinedForSameCell_DeleteFirst() throws IOException {
    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"),
                                     0L, Type.Delete);
    KeyValue toAdd = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                  Bytes.toBytes("_qual_"),
                                  Bytes.toBytes("value"));
    RowData deleteEventRowData = createEventRowData("_row_", toDelete);
    RowData addEventRowData = createEventRowData("_row_", toAdd);

    indexer.calculateIndexUpdates(
            ImmutableList.of(deleteEventRowData, addEventRowData), updateCollector);

    assertTrue(updateCollector.getIdsToDelete().isEmpty());
    List<SolrInputDocument> documents = Lists.newArrayList(updateCollector.getDocumentsToAdd().values());
    assertEquals(1, documents.size());
    assertEquals("_row_-_cf_-_qual_", documents.get(0).getFieldValue("id"));
}
 
Example 16
Source Project: phoenix   Source File: IndexMaintainer.java    License: Apache License 2.0 6 votes vote down vote up
private DeleteType getDeleteTypeOrNull(Collection<? extends Cell> pendingUpdates, int nCFs) {
      int nDeleteCF = 0;
      int nDeleteVersionCF = 0;
      for (Cell kv : pendingUpdates) {
      	if (kv.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()) {
              nDeleteVersionCF++;
          }
      	else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
      			// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
      	        || TransactionUtil.isDeleteFamily(kv)) {
      	    nDeleteCF++;
      	}
      }
      // This is what a delete looks like on the server side for mutable indexing...
      // Should all be one or the other for DeleteFamily versus DeleteFamilyVersion, but just in case not
      DeleteType deleteType = null; 
      if (nDeleteVersionCF > 0 && nDeleteVersionCF >= nCFs) {
      	deleteType = DeleteType.SINGLE_VERSION;
      } else {
	int nDelete = nDeleteCF + nDeleteVersionCF;
	if (nDelete>0 && nDelete >= nCFs) {
		deleteType = DeleteType.ALL_VERSIONS;
	}
}
      return deleteType;
  }
 
Example 17
Source Project: phoenix   Source File: IndexMaintainer.java    License: Apache License 2.0 5 votes vote down vote up
private boolean hasIndexedColumnChanged(ValueGetter oldState, Collection<KeyValue> pendingUpdates) throws IOException {
    if (pendingUpdates.isEmpty()) {
        return false;
    }
    Map<ColumnReference,KeyValue> newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); 
    for (KeyValue kv : pendingUpdates) {
        newState.put(new ColumnReference(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv)), kv);
    }
    for (ColumnReference ref : indexedColumns) {
    	KeyValue newValue = newState.get(ref);
    	if (newValue != null) { // Indexed column has potentially changed
    		ImmutableBytesPtr oldValue = oldState.getLatestValue(ref);
    		boolean newValueSetAsNull = newValue.getTypeByte() == Type.DeleteColumn.getCode();
    		//If the new column value has to be set as null and the older value is null too,
    		//then just skip to the next indexed column.
    		if (newValueSetAsNull && oldValue == null) {
    			continue;
    		}
    		if ((oldValue == null && !newValueSetAsNull) || (oldValue != null && newValueSetAsNull)) {
    			return true;
    		}
    		// If the old value is different than the new value, the index row needs to be deleted
    		if (Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), 
    				newValue.getValueArray(), newValue.getValueOffset(), newValue.getValueLength()) != 0) {
    			return true;
    		}
    	}
    }
    return false;
}
 
Example 18
Source Project: phoenix   Source File: KeyValueUtil.java    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public static KeyValue newKeyValue(byte[] key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, 0, key.length,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example 19
Source Project: phoenix   Source File: KeyValueUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static KeyValue newKeyValue(byte[] key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, 0, key.length,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example 20
/**
 * Test that when we do a column delete at a given timestamp that we delete the entire column.
 * @throws Exception
 */
@Test
public void testCoverForDeleteColumn() throws Exception {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue dc = createKvForType(Type.DeleteColumn, 11);
  KeyValue put = createKvForType(Type.Put, 10);
  assertEquals("Didn't filter out delete column.", ReturnCode.SKIP, filter.filterKeyValue(dc));
  assertEquals("Didn't get a seek hint for the deleted column", ReturnCode.SEEK_NEXT_USING_HINT,
    filter.filterKeyValue(put));
  // seek past the given put
  KeyValue seek = filter.getNextKeyHint(put);
  assertTrue("Seeked key wasn't past the expected put - didn't skip the column",
    KeyValue.COMPARATOR.compare(seek, put) > 0);
}
 
Example 21
Source Project: phoenix   Source File: KeyValueUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static KeyValue newKeyValue(byte[] key, int keyOffset, int keyLength, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, keyOffset, keyLength,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example 22
Source Project: phoenix   Source File: ValueGetterTuple.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public KeyValue getValue(byte[] family, byte[] qualifier) {
	ImmutableBytesPtr value = null;
    try {
        value = valueGetter.getLatestValue(new ColumnReference(family, qualifier));
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
	return new KeyValue(valueGetter.getRowKey(), family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put, value!=null? copyBytesIfNecessary(value) : null);
}
 
Example 23
Source Project: phoenix   Source File: TestIndexMemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * We don't expect custom KeyValue creation, so we can't get into weird situations, where a
 * {@link Type#DeleteFamily} has a column qualifier specified.
 * @throws Exception
 */
@Test
public void testExpectedOrdering() throws Exception {
  IndexMemStore store = new IndexMemStore();
  KeyValue kv = new KeyValue(row, family, qual, 12, Type.Put, val);
  store.add(kv, true);
  KeyValue kv2 = new KeyValue(row, family, qual, 10, Type.Put, val2);
  store.add(kv2, true);
  KeyValue df = new KeyValue(row, family, null, 11, Type.DeleteFamily, null);
  store.add(df, true);
  KeyValue dc = new KeyValue(row, family, qual, 11, Type.DeleteColumn, null);
  store.add(dc, true);
  KeyValue d = new KeyValue(row, family, qual, 12, Type.Delete, null);
  store.add(d, true);

  // null qualifiers should always sort before the non-null cases
  KeyValueScanner scanner = store.getScanner();
  KeyValue first = KeyValue.createFirstOnRow(row);
  assertTrue("Didn't have any data in the scanner", scanner.seek(first));
  assertTrue("Didn't get delete family first (no qualifier == sort first)", df == scanner.next());
  assertTrue("Didn't get point delete before corresponding put", d == scanner.next());
  assertTrue("Didn't get larger ts Put", kv == scanner.next());
  assertTrue("Didn't get delete column before corresponding put(delete sorts first)",
    dc == scanner.next());
  assertTrue("Didn't get smaller ts Put", kv2 == scanner.next());
  assertNull("Have more data in the scanner", scanner.next());
}
 
Example 24
Source Project: phoenix   Source File: TestApplyAndFilterDeletesFilter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testDeletesAreNotReturned() {
  KeyValue kv = createKvForType(Type.Delete);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  assertEquals("Didn't skip point delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteColumn);
  assertEquals("Didn't skip from column delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteFamily);
  assertEquals("Didn't skip from family delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));
}
 
Example 25
Source Project: phoenix   Source File: TestApplyAndFilterDeletesFilter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Hinting with this filter is a little convoluted as we binary search the list of families to
 * attempt to find the right one to seek.
 */
@Test
public void testHintCorrectlyToNextFamily() {
  // start with doing a family delete, so we will seek to the next column
  KeyValue kv = createKvForType(Type.DeleteFamily);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  assertEquals(ReturnCode.SKIP, filter.filterKeyValue(kv));
  KeyValue next = createKvForType(Type.Put);
  // make sure the hint is our attempt at the end key, because we have no more families to seek
  assertEquals("Didn't get a hint from a family delete", ReturnCode.SEEK_NEXT_USING_HINT,
    filter.filterKeyValue(next));
  assertEquals("Didn't get END_KEY with no families to match", KeyValue.LOWESTKEY,
    filter.getNextCellHint(next));

  // check for a family that comes before our family, so we always seek to the end as well
  filter = new ApplyAndFilterDeletesFilter(asSet(Bytes.toBytes("afamily")));
  assertEquals(ReturnCode.SKIP, filter.filterKeyValue(kv));
  // make sure the hint is our attempt at the end key, because we have no more families to seek
  assertEquals("Didn't get a hint from a family delete", ReturnCode.SEEK_NEXT_USING_HINT,
    filter.filterKeyValue(next));
  assertEquals("Didn't get END_KEY with no families to match", KeyValue.LOWESTKEY,
    filter.getNextCellHint(next));

  // check that we seek to the correct family that comes after our family
  byte[] laterFamily = Bytes.toBytes("zfamily");
  filter = new ApplyAndFilterDeletesFilter(asSet(laterFamily));
  assertEquals(ReturnCode.SKIP, filter.filterKeyValue(kv));
  @SuppressWarnings("deprecation")
  KeyValue expected = KeyValue.createFirstOnRow(kv.getRow(), laterFamily, new byte[0]);
  assertEquals("Didn't get a hint from a family delete", ReturnCode.SEEK_NEXT_USING_HINT,
    filter.filterKeyValue(next));
  assertEquals("Didn't get correct next key with a next family", expected,
    filter.getNextCellHint(next));
}
 
Example 26
@Test
public void testDeletesAreNotReturned() {
  KeyValue kv = createKvForType(Type.Delete);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  assertEquals("Didn't skip point delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteColumn);
  assertEquals("Didn't skip from column delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteFamily);
  assertEquals("Didn't skip from family delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));
}
 
Example 27
@Test
public void testReadWrite() throws IOException {
  byte[] row = Bytes.toBytes("row");
  byte[] family = Bytes.toBytes("family");
  byte[] qualifier = Bytes.toBytes("qualifier");
  byte[] value = Bytes.toBytes("value");
  long ts = 10;
  Type type = KeyValue.Type.Put;
  ClientKeyValue kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type,
      wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);

  type = Type.Delete;
  kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type, wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);

  type = Type.DeleteColumn;
  kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type, wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);

  type = Type.DeleteFamily;
  kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type, wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);

  type = Type.Maximum;
  kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type, wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);

  // test a couple different variables, to make sure we aren't faking it
  row = Bytes.toBytes("row-never-seen-before1234");
  family = Bytes.toBytes("family-to-test-more");
  qualifier = Bytes.toBytes("untested-qualifier");
  value = Bytes.toBytes("value-that-we-haven't_tested");
  ts = System.currentTimeMillis();
  kv = new ClientKeyValue(wrap(row), wrap(family), wrap(qualifier), ts, type, wrap(value));
  validate(kv, row, family, qualifier, ts, type, value);
}
 
Example 28
Source Project: phoenix   Source File: TestLocalTableState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];
      KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
      kv.setSequenceId(0);
      list.add(kv);
      return false;
    }
  });


  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  //add the kvs from the mutation
  table.addPendingUpdates(KeyValueUtil.ensureKeyValues(m.get(fam, qual)));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 
Example 29
/**
 * Validate that a single-version delete is not used. Having to do this for now because transactional delete
 * mechansim will currently treat DeleteColumn the same as Delete which could cause confusion.
 */
public static void validateDelete(final Delete delete) throws SingleVersionDeleteNotSupported {
    Collection<List<KeyValue>> values = delete.getFamilyMap().values();
    for (List<KeyValue> value : values) {
        for (KeyValue kv : value) {
            if (Type.Delete.getCode() == kv.getType()) {
                throw new SingleVersionDeleteNotSupported();
            }
        }
    }
}
 
Example 30
Source Project: hbase   Source File: VisibilityScanDeleteTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void add(Cell delCell) {
  //Cannot call super.add because need to find if the delete needs to be considered
  long timestamp = delCell.getTimestamp();
  byte type = delCell.getTypeByte();
  if (type == KeyValue.Type.DeleteFamily.getCode()) {
    hasFamilyStamp = true;
    boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
    if (!hasVisTag && timestamp > familyStamp) {
      familyStamp = timestamp;
    }
    return;
  } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
    familyVersionStamps.add(timestamp);
    extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
    return;
  }
  // new column, or more general delete type
  if (deleteCell != null) {
    if (!(CellUtil.matchingQualifier(delCell, deleteCell))) {
      // A case where there are deletes for a column qualifier but there are
      // no corresponding puts for them. Rare case.
      visibilityTagsDeleteColumns = null;
      visiblityTagsDeleteColumnVersion = null;
    } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
      // there is a timestamp change which means we could clear the list
      // when ts is same and the vis tags are different we need to collect
      // them all. Interesting part is that in the normal case of puts if
      // there are 2 cells with same ts and diff vis tags only one of them is
      // returned. Handling with a single List<Tag> would mean that only one
      // of the cell would be considered. Doing this as a precaution.
      // Rare cases.
      visiblityTagsDeleteColumnVersion = null;
    }
  }
  deleteCell = delCell;
  deleteType = type;
  deleteTimestamp = timestamp;
  extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
}