org.apache.hadoop.hbase.KeyValue.Type Java Examples

The following examples show how to use org.apache.hadoop.hbase.KeyValue.Type. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestHBaseInterClusterReplicationEndpointFilterEdits.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testFilterNotExistColumnFamilyEdits() {
  List<List<Entry>> entryList = new ArrayList<>();
  // should be filtered
  Cell c1 = new KeyValue(ROW, NON_EXISTING_FAMILY, QUALIFIER, System.currentTimeMillis(),
      Type.Put, VALUE);
  Entry e1 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c1));
  entryList.add(Lists.newArrayList(e1));
  // should be kept
  Cell c2 = new KeyValue(ROW, FAMILY, QUALIFIER, System.currentTimeMillis(), Type.Put, VALUE);
  Entry e2 = new Entry(new WALKeyImpl(new byte[32], TABLE1, System.currentTimeMillis()),
      new WALEdit().add(c2));
  entryList.add(Lists.newArrayList(e2, e1));
  List<List<Entry>> filtered = endpoint.filterNotExistColumnFamilyEdits(entryList);
  assertEquals(1, filtered.size());
  assertEquals(1, filtered.get(0).get(0).getEdit().getCells().size());
  Cell cell = filtered.get(0).get(0).getEdit().getCells().get(0);
  assertTrue(CellUtil.matchingFamily(cell, FAMILY));
}
 
Example #2
Source File: IndexMaintainer.java    From phoenix with Apache License 2.0 6 votes vote down vote up
public boolean isRowDeleted(Collection<KeyValue> pendingUpdates) {
    int nDeleteCF = 0;
    for (KeyValue kv : pendingUpdates) {
        if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()) {
            nDeleteCF++;
            boolean isEmptyCF = Bytes.compareTo(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), 
              dataEmptyKeyValueCF, 0, dataEmptyKeyValueCF.length) == 0;
            // This is what a delete looks like on the client side for immutable indexing...
            if (isEmptyCF) {
                return true;
            }
        }
    }
    // This is what a delete looks like on the server side for mutable indexing...
    return nDeleteCF == this.nDataCFs;
}
 
Example #3
Source File: IndexMaintainer.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private DeleteType getDeleteTypeOrNull(Collection<? extends Cell> pendingUpdates, int nCFs) {
      int nDeleteCF = 0;
      int nDeleteVersionCF = 0;
      for (Cell kv : pendingUpdates) {
      	if (kv.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()) {
              nDeleteVersionCF++;
          }
      	else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
      			// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
      	        || TransactionUtil.isDeleteFamily(kv)) {
      	    nDeleteCF++;
      	}
      }
      // This is what a delete looks like on the server side for mutable indexing...
      // Should all be one or the other for DeleteFamily versus DeleteFamilyVersion, but just in case not
      DeleteType deleteType = null; 
      if (nDeleteVersionCF > 0 && nDeleteVersionCF >= nCFs) {
      	deleteType = DeleteType.SINGLE_VERSION;
      } else {
	int nDelete = nDeleteCF + nDeleteVersionCF;
	if (nDelete>0 && nDelete >= nCFs) {
		deleteType = DeleteType.ALL_VERSIONS;
	}
}
      return deleteType;
  }
 
Example #4
Source File: ColumnBasedIndexerTest.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_UpdateAndDeleteCombinedForSameCell_DeleteFirst() throws IOException {
    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"),
                                     0L, Type.Delete);
    KeyValue toAdd = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                  Bytes.toBytes("_qual_"),
                                  Bytes.toBytes("value"));
    RowData deleteEventRowData = createEventRowData("_row_", toDelete);
    RowData addEventRowData = createEventRowData("_row_", toAdd);

    indexer.calculateIndexUpdates(
            ImmutableList.of(deleteEventRowData, addEventRowData), updateCollector);

    assertTrue(updateCollector.getIdsToDelete().isEmpty());
    List<SolrInputDocument> documents = Lists.newArrayList(updateCollector.getDocumentsToAdd().values());
    assertEquals(1, documents.size());
    assertEquals("_row_-_cf_-_qual_", documents.get(0).getFieldValue("id"));
}
 
Example #5
Source File: ColumnBasedIndexerTest.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_DeleteFamily() throws IOException {

    final String ROW_FIELD = "_row_field_";
    final String FAMILY_FIELD = "_family_field_";

    doReturn(ROW_FIELD).when(indexerConf).getRowField();
    doReturn(FAMILY_FIELD).when(indexerConf).getColumnFamilyField();

    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"), 0L, Type.DeleteFamily);
    RowData rowData = createEventRowData("_row_", toDelete);

    indexer.calculateIndexUpdates(Lists.newArrayList(rowData), updateCollector);

    assertEquals(ImmutableList.of("(" + ROW_FIELD + ":_row_)AND(" + FAMILY_FIELD + ":_cf_)"),
            updateCollector.getDeleteQueries());
    assertTrue(updateCollector.getIdsToDelete().isEmpty());
    assertTrue(updateCollector.getDocumentsToAdd().isEmpty());
}
 
Example #6
Source File: SchemaUtil.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private static KeyValue upgradeTo3(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    int newLength = keyValue.getRowLength() + 1;
    byte[] newKey = new byte[newLength];
    newKey[0] = QueryConstants.SEPARATOR_BYTE;
    System.arraycopy(buf, keyValue.getRowOffset(), newKey, 1, keyValue.getRowLength());
    byte[] valueBuf = updateValueIfNecessary(keyValue);
    int valueOffset = keyValue.getValueOffset();
    int valueLength = keyValue.getValueLength();
    if (valueBuf != buf) {
        valueOffset = 0;
        valueLength = valueBuf.length;
    }
    return new KeyValue(newKey, 0, newLength,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), Type.codeToType(keyValue.getType()),
            valueBuf, valueOffset, valueLength);
}
 
Example #7
Source File: TestClientKeyValueLocal.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
private void validate(KeyValue kv, byte[] row, byte[] family, byte[] qualifier, long ts,
    Type type, byte[] value) throws IOException {
  DataOutputBuffer out = new DataOutputBuffer();
  kv.write(out);
  out.close();
  byte[] data = out.getData();
  // read it back in
  KeyValue read = new KeyValue();
  DataInputBuffer in = new DataInputBuffer();
  in.reset(data, data.length);
  read.readFields(in);
  in.close();

  // validate that its the same
  assertTrue("Row didn't match!", Bytes.equals(row, read.getRow()));
  assertTrue("Family didn't match!", Bytes.equals(family, read.getFamily()));
  assertTrue("Qualifier didn't match!", Bytes.equals(qualifier, read.getQualifier()));
  assertTrue("Value didn't match!", Bytes.equals(value, read.getValue()));
  assertEquals("Timestamp didn't match", ts, read.getTimestamp());
  assertEquals("Type didn't match", type.getCode(), read.getType());
}
 
Example #8
Source File: ColumnProjectionFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void filterRowCells(List<Cell> kvs) throws IOException {
    if (kvs.isEmpty()) return;
    Cell firstKV = kvs.get(0);
    Iterator<Cell> itr = kvs.iterator();
    while (itr.hasNext()) {
        Cell kv = itr.next();
        ptr.set(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength());
        if (this.columnsTracker.containsKey(ptr)) {
            Set<ImmutableBytesPtr> cols = this.columnsTracker.get(ptr);
            ptr.set(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
            if (cols != null && !(cols.contains(ptr))) {
                itr.remove();
            }
        } else {
            itr.remove();
        }
    }
    // make sure we're not holding to any of the byte[]'s
    ptr.set(HConstants.EMPTY_BYTE_ARRAY);
    if (kvs.isEmpty()) {
        kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(),firstKV.getRowLength(), this.emptyCFName,
                0, this.emptyCFName.length, QueryConstants.EMPTY_COLUMN_BYTES, 0,
                QueryConstants.EMPTY_COLUMN_BYTES.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0));
    }
}
 
Example #9
Source File: TestIndexMemStore.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR);
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  KeyValueScanner scanner = store.getScanner();
  KeyValue first = KeyValue.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 
Example #10
Source File: TestClientKeyValueLocal.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * Copied from {@link TestKeyValue}
 * @throws Exception
 */
@Test
public void testColumnCompare() throws Exception {
  final byte [] a = Bytes.toBytes("aaa");
  byte [] family1 = Bytes.toBytes("abc");
  byte [] qualifier1 = Bytes.toBytes("def");
  byte [] family2 = Bytes.toBytes("abcd");
  byte [] qualifier2 = Bytes.toBytes("ef");

  KeyValue aaa = new ClientKeyValue(a, family1, qualifier1, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family2, qualifier2));
  assertTrue(aaa.matchingColumn(family1, qualifier1));
  aaa = new ClientKeyValue(a, family2, qualifier2, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family1, qualifier1));
  assertTrue(aaa.matchingColumn(family2,qualifier2));
  byte [] nullQualifier = new byte[0];
  aaa = new ClientKeyValue(a, family1, nullQualifier, 0L, Type.Put, a);
  assertTrue(aaa.matchingColumn(family1,null));
  assertFalse(aaa.matchingColumn(family2,qualifier2));
}
 
Example #11
Source File: ColumnBasedIndexerTest.java    From hbase-indexer with Apache License 2.0 6 votes vote down vote up
@Test
public void testCalculateIndexUpdates_UpdateAndDeleteCombinedForSameCell_UpdateFirst() throws IOException {
    KeyValue toAdd = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                  Bytes.toBytes("_qual_"),
                                  Bytes.toBytes("value"));
    KeyValue toDelete = new KeyValue(Bytes.toBytes("_row_"), Bytes.toBytes("_cf_"),
                                     Bytes.toBytes("_qual_"),
                                     0L, Type.DeleteColumn);
    RowData addEventRowData = createEventRowData("_row_", toAdd);
    RowData deleteEventRowData = createEventRowData("_row_", toDelete);

    indexer.calculateIndexUpdates(ImmutableList.of(addEventRowData, deleteEventRowData), updateCollector);

    assertEquals(Lists.newArrayList("_row_-_cf_-_qual_"), updateCollector.getIdsToDelete());
    assertTrue(updateCollector.getDocumentsToAdd().isEmpty());
}
 
Example #12
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * Test that we don't cover other columns when we have a delete column.
 */
@Test
public void testDeleteColumnCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue d = createKvForType(Type.DeleteColumn, 12);
  byte[] qual2 = Bytes.add(qualifier, Bytes.toBytes("-other"));
  KeyValue put =
      new KeyValue(row, family, qual2, 0, qual2.length, 11, Type.Put, value, 0,
          value.length);

  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // different column put should still be visible
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // set a delete family, but in the past
  d = createKvForType(Type.DeleteFamily, 10);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // add back in the original delete column
  d = createKvForType(Type.DeleteColumn, 11);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // onto a different family, so that must be visible too
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));
}
 
Example #13
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * DeleteFamily markers should delete everything from that timestamp backwards, but not hide
 * anything forwards
 */
@Test
public void testDeleteFamilyCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue df = createKvForType(Type.DeleteFamily, 11);
  KeyValue put = createKvForType(Type.Put, 12);

  assertEquals("Didn't filter out delete family", ReturnCode.SKIP, filter.filterKeyValue(df));
  assertEquals("Filtered out put with newer TS than delete family", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // older kv shouldn't be visible
  put = createKvForType(Type.Put, 10);
  assertEquals("Didn't filter out older put, covered by DeleteFamily marker",
    ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(put));

  // next seek should be past the families
  assertEquals(KeyValue.LOWESTKEY, filter.getNextCellHint(put));
}
 
Example #14
Source File: TestCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example #15
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Point deletes should only cover the exact entry they are tied to. Earlier puts should always
 * show up.
 */
@Test
public void testCoveringPointDelete() {
  // start with doing a family delete, so we will seek to the next column
  KeyValue kv = createKvForType(Type.Delete);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  filter.filterKeyValue(kv);
  KeyValue put = createKvForType(Type.Put);
  assertEquals("Didn't filter out put with same timestamp!", ReturnCode.SKIP,
    filter.filterKeyValue(put));
  // we should filter out the exact same put again, which could occur with the kvs all kept in the
  // same memstore
  assertEquals("Didn't filter out put with same timestamp on second call!", ReturnCode.SKIP,
    filter.filterKeyValue(put));

  // ensure then that we don't filter out a put with an earlier timestamp (though everything else
  // matches)
  put = createKvForType(Type.Put, ts - 1);
  assertEquals("Didn't accept put that has an earlier ts than the covering delete!",
    ReturnCode.INCLUDE, filter.filterKeyValue(put));
}
 
Example #16
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
/**
 * DeleteFamily markers should delete everything from that timestamp backwards, but not hide
 * anything forwards
 */
@Test
public void testDeleteFamilyCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue df = createKvForType(Type.DeleteFamily, 11);
  KeyValue put = createKvForType(Type.Put, 12);

  assertEquals("Didn't filter out delete family", ReturnCode.SKIP, filter.filterKeyValue(df));
  assertEquals("Filtered out put with newer TS than delete family", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // older kv shouldn't be visible
  put = createKvForType(Type.Put, 10);
  assertEquals("Didn't filter out older put, covered by DeleteFamily marker",
    ReturnCode.SEEK_NEXT_USING_HINT, filter.filterKeyValue(put));

  // next seek should be past the families
  assertEquals(KeyValue.LOWESTKEY, filter.getNextKeyHint(put));
}
 
Example #17
Source File: LocalTableStateTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);


  Put pendingUpdate = new Put(row);
  pendingUpdate.addColumn(fam, qual, ts, val);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, pendingUpdate);


  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
}
 
Example #18
Source File: TestByteBufferKeyValue.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testByteBufferBackedKeyValueWithTags() throws Exception {
  KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0L, Type.Put, row1, tags);
  ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
  ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(buf, 0, buf.capacity(), 0L);
  assertEquals(
    ROW1,
    ByteBufferUtils.toStringBinary(offheapKV.getRowByteBuffer(),
      offheapKV.getRowPosition(), offheapKV.getRowLength()));
  assertEquals(
    FAM1,
    ByteBufferUtils.toStringBinary(offheapKV.getFamilyByteBuffer(),
      offheapKV.getFamilyPosition(), offheapKV.getFamilyLength()));
  assertEquals(
    QUAL1,
    ByteBufferUtils.toStringBinary(offheapKV.getQualifierByteBuffer(),
      offheapKV.getQualifierPosition(), offheapKV.getQualifierLength()));
  assertEquals(
    ROW1,
    ByteBufferUtils.toStringBinary(offheapKV.getValueByteBuffer(),
      offheapKV.getValuePosition(), offheapKV.getValueLength()));
  assertEquals(0L, offheapKV.getTimestamp());
  assertEquals(Type.Put.getCode(), offheapKV.getTypeByte());
  // change tags to handle both onheap and offheap stuff
  List<Tag> resTags = PrivateCellUtil.getTags(offheapKV);
  Tag tag1 = resTags.get(0);
  assertEquals(t1.getType(), tag1.getType());
  assertEquals(Tag.getValueAsString(t1),
    Tag.getValueAsString(tag1));
  Tag tag2 = resTags.get(1);
  assertEquals(tag2.getType(), tag2.getType());
  assertEquals(Tag.getValueAsString(t2),
    Tag.getValueAsString(tag2));
  Tag res = PrivateCellUtil.getTag(offheapKV, (byte) 2).get();
  assertEquals(Tag.getValueAsString(t2),
    Tag.getValueAsString(tag2));
  assertFalse(PrivateCellUtil.getTag(offheapKV, (byte) 3).isPresent());
}
 
Example #19
Source File: TestCellComparator.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testCompareCellWithKey() throws Exception {
  KeyValue kv1 = new KeyValue(row1, fam1, qual1, val);
  KeyValue kv2 = new KeyValue(row2, fam1, qual1, val);
  assertTrue(
    (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) < 0);

  kv1 = new KeyValue(row1, fam2, qual1, val);
  kv2 = new KeyValue(row1, fam1, qual1, val);
  assertTrue(
    (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0);

  kv1 = new KeyValue(row1, fam1, qual1, 1L, val);
  kv2 = new KeyValue(row1, fam1, qual1, 2L, val);
  assertTrue(
    (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0);

  kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put);
  kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Maximum);
  assertTrue(
    (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) > 0);

  kv1 = new KeyValue(row1, fam1, qual1, 1L, Type.Put);
  kv2 = new KeyValue(row1, fam1, qual1, 1L, Type.Put);
  assertTrue(
    (PrivateCellUtil.compare(comparator, kv1, kv2.getKey(), 0, kv2.getKey().length)) == 0);
}
 
Example #20
Source File: TestApplyAndFilterDeletesFilter.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeletesAreNotReturned() {
  KeyValue kv = createKvForType(Type.Delete);
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  assertEquals("Didn't skip point delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteColumn);
  assertEquals("Didn't skip from column delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));

  filter.reset();
  kv = createKvForType(Type.DeleteFamily);
  assertEquals("Didn't skip from family delete!", ReturnCode.SKIP, filter.filterKeyValue(kv));
}
 
Example #21
Source File: BufferedDataBlockEncoder.java    From hbase with Apache License 2.0 5 votes vote down vote up
private int compareTypeBytes(Cell key, Cell right) {
  if (key.getFamilyLength() + key.getQualifierLength() == 0
      && key.getTypeByte() == Type.Minimum.getCode()) {
    // left is "bigger", i.e. it appears later in the sorted order
    return 1;
  }
  if (right.getFamilyLength() + right.getQualifierLength() == 0
      && right.getTypeByte() == Type.Minimum.getCode()) {
    return -1;
  }
  return 0;
}
 
Example #22
Source File: KeyValueUtil.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public static KeyValue newKeyValue(Result r, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    byte[] bytes = ResultUtil.getRawBytes(r);
    return new KeyValue(bytes, ResultUtil.getKeyOffset(r), ResultUtil.getKeyLength(r),
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example #23
Source File: TestHFile.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Cell getCell(byte[] row, byte[] family, byte[] qualifier) {
  return ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(row)
    .setFamily(family)
    .setQualifier(qualifier)
    .setTimestamp(HConstants.LATEST_TIMESTAMP)
    .setType(KeyValue.Type.Maximum.getCode())
    .setValue(HConstants.EMPTY_BYTE_ARRAY)
    .build();
}
 
Example #24
Source File: KeyValueUtil.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public static KeyValue newKeyValue(ImmutableBytesWritable key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key.get(), key.getOffset(), key.getLength(),
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 
Example #25
Source File: TestHFile.java    From hbase with Apache License 2.0 5 votes vote down vote up
public static KeyValue.Type generateKeyType(Random rand) {
  if (rand.nextBoolean()) {
    // Let's make half of KVs puts.
    return KeyValue.Type.Put;
  } else {
    KeyValue.Type keyType = KeyValue.Type.values()[1 + rand.nextInt(NUM_VALID_KEY_TYPES)];
    if (keyType == KeyValue.Type.Minimum || keyType == KeyValue.Type.Maximum) {
      throw new RuntimeException("Generated an invalid key type: " + keyType + ". "
          + "Probably the layout of KeyValue.Type has changed.");
    }
    return keyType;
  }
}
 
Example #26
Source File: IndexMaintainer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private boolean hasIndexedColumnChanged(ValueGetter oldState, Collection<? extends Cell> pendingUpdates, long ts) throws IOException {
    if (pendingUpdates.isEmpty()) {
        return false;
    }
    Map<ColumnReference,Cell> newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); 
    for (Cell kv : pendingUpdates) {
        newState.put(new ColumnReference(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv)), kv);
    }
    for (ColumnReference ref : indexedColumns) {
    	Cell newValue = newState.get(ref);
    	if (newValue != null) { // Indexed column has potentially changed
    	    ImmutableBytesWritable oldValue = oldState.getLatestValue(ref, ts);
            boolean newValueSetAsNull = (newValue.getTypeByte() == Type.DeleteColumn.getCode() || newValue.getTypeByte() == Type.Delete.getCode() || CellUtil.matchingValue(newValue, HConstants.EMPTY_BYTE_ARRAY));
    		boolean oldValueSetAsNull = oldValue == null || oldValue.getLength() == 0;
    		//If the new column value has to be set as null and the older value is null too,
    		//then just skip to the next indexed column.
    		if (newValueSetAsNull && oldValueSetAsNull) {
    			continue;
    		}
    		if (oldValueSetAsNull || newValueSetAsNull) {
    			return true;
    		}
    		// If the old value is different than the new value, the index row needs to be deleted
    		if (Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), 
    				newValue.getValueArray(), newValue.getValueOffset(), newValue.getValueLength()) != 0) {
    			return true;
    		}
    	}
    }
    return false;
}
 
Example #27
Source File: TestIndexMemStore.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * We don't expect custom KeyValue creation, so we can't get into weird situations, where a
 * {@link Type#DeleteFamily} has a column qualifier specified.
 * @throws Exception
 */
@Test
public void testExpectedOrdering() throws Exception {
  IndexMemStore store = new IndexMemStore();
  KeyValue kv = new KeyValue(row, family, qual, 12, Type.Put, val);
  store.add(kv, true);
  KeyValue kv2 = new KeyValue(row, family, qual, 10, Type.Put, val2);
  store.add(kv2, true);
  KeyValue df = new KeyValue(row, family, null, 11, Type.DeleteFamily, null);
  store.add(df, true);
  KeyValue dc = new KeyValue(row, family, qual, 11, Type.DeleteColumn, null);
  store.add(dc, true);
  KeyValue d = new KeyValue(row, family, qual, 12, Type.Delete, null);
  store.add(d, true);

  // null qualifiers should always sort before the non-null cases
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  assertTrue("Didn't have any data in the scanner", scanner.seek(first));
  assertTrue("Didn't get delete family first (no qualifier == sort first)", df == scanner.next());
  assertTrue("Didn't get point delete before corresponding put", d == scanner.next());
  assertTrue("Didn't get larger ts Put", kv == scanner.next());
  assertTrue("Didn't get delete column before corresponding put(delete sorts first)",
    dc == scanner.next());
  assertTrue("Didn't get smaller ts Put", kv2 == scanner.next());
  assertNull("Have more data in the scanner", scanner.next());
}
 
Example #28
Source File: TestClientKeyValueLocal.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
/**
 * Test a corner case when the family qualifier is a prefix of the column qualifier.
 */
@Test
public void testColumnCompare_prefix() throws Exception {
  final byte[] a = Bytes.toBytes("aaa");
  byte[] family1 = Bytes.toBytes("abc");
  byte[] qualifier1 = Bytes.toBytes("def");
  byte[] family2 = Bytes.toBytes("ab");
  byte[] qualifier2 = Bytes.toBytes("def");

  KeyValue aaa = new ClientKeyValue(a, family1, qualifier1, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family2, qualifier2));
}
 
Example #29
Source File: ColumnProjectionFilter.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void filterRowCells(List<Cell> kvs) throws IOException {
    if (kvs.isEmpty()) return;
    Cell firstKV = kvs.get(0);
    Iterables.removeIf(kvs, new Predicate<Cell>() {
        @Override
        public boolean apply(Cell kv) {
            ptr.set(kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength());
            if (columnsTracker.containsKey(ptr)) {
                Set<ImmutableBytesPtr> cols = columnsTracker.get(ptr);
                ptr.set(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength());
                if (cols != null && !(cols.contains(ptr))) {
                    return true;
                }
            } else {
                return true;
            }
            return false;
        }
    });
    // make sure we're not holding to any of the byte[]'s
    ptr.set(HConstants.EMPTY_BYTE_ARRAY);
    if (kvs.isEmpty()) {
        kvs.add(new KeyValue(firstKV.getRowArray(), firstKV.getRowOffset(), firstKV.getRowLength(),
                this.emptyCFName, 0, this.emptyCFName.length, emptyKVQualifier, 0,
                emptyKVQualifier.length, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0));
    }
}
 
Example #30
Source File: KeyValueUtil.java    From phoenix with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public static KeyValue newKeyValue(byte[] key, int keyOffset, int keyLength, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, keyOffset, keyLength,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}