Java Code Examples for org.apache.hadoop.hbase.KeyValue.Type#Put

@Test
public void testGetKeyMethods() throws Exception {
  KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0L, Type.Put, row1, tags);
  ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getKeyLength());
  ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), kvCell.getKeyOffset(),
    kvCell.getKeyLength());
  ByteBufferExtendedCell offheapKeyOnlyKV = new ByteBufferKeyOnlyKeyValue(buf, 0, buf.capacity());
  assertEquals(
    ROW1,
    ByteBufferUtils.toStringBinary(offheapKeyOnlyKV.getRowByteBuffer(),
      offheapKeyOnlyKV.getRowPosition(), offheapKeyOnlyKV.getRowLength()));
  assertEquals(
    FAM1,
    ByteBufferUtils.toStringBinary(offheapKeyOnlyKV.getFamilyByteBuffer(),
      offheapKeyOnlyKV.getFamilyPosition(), offheapKeyOnlyKV.getFamilyLength()));
  assertEquals(
    QUAL1,
    ByteBufferUtils.toStringBinary(offheapKeyOnlyKV.getQualifierByteBuffer(),
      offheapKeyOnlyKV.getQualifierPosition(),
      offheapKeyOnlyKV.getQualifierLength()));
  assertEquals(0L, offheapKeyOnlyKV.getTimestamp());
  assertEquals(Type.Put.getCode(), offheapKeyOnlyKV.getTypeByte());
}
 

@Test
public void testToString() {

    SingleKeyValueTuple singleKeyValueTuple = new SingleKeyValueTuple();
    assertTrue(singleKeyValueTuple.toString().equals("SingleKeyValueTuple[null]"));
    final byte [] rowKey = Bytes.toBytes("aaa");
    singleKeyValueTuple.setKey(new ImmutableBytesWritable(rowKey));
    assertTrue(singleKeyValueTuple.toString().equals("SingleKeyValueTuple[aaa]"));

    byte [] family1 = Bytes.toBytes("abc");
    byte [] qualifier1 = Bytes.toBytes("def");
    KeyValue keyValue = new KeyValue(rowKey, family1, qualifier1, 0L, Type.Put, rowKey);
    singleKeyValueTuple = new SingleKeyValueTuple(keyValue);
    assertTrue(singleKeyValueTuple.toString().startsWith("SingleKeyValueTuple[aaa/abc:def/0/Put/vlen=3"));
    assertTrue(singleKeyValueTuple.toString().endsWith("]"));
}
 

@Test
public void testRowIndexWithTagsButNoTagsInCell() throws IOException {
  List<KeyValue> kvList = new ArrayList<>();
  byte[] row = new byte[0];
  byte[] family = new byte[0];
  byte[] qualifier = new byte[0];
  byte[] value = new byte[0];
  KeyValue expectedKV = new KeyValue(row, family, qualifier, 1L, Type.Put, value);
  kvList.add(expectedKV);
  DataBlockEncoding encoding = DataBlockEncoding.ROW_INDEX_V1;
  DataBlockEncoder encoder = encoding.getEncoder();
  ByteBuffer encodedBuffer =
      encodeKeyValues(encoding, kvList, getEncodingContext(Algorithm.NONE, encoding), false);
  HFileContext meta =
      new HFileContextBuilder().withHBaseCheckSum(false).withIncludesMvcc(includesMemstoreTS)
          .withIncludesTags(includesTags).withCompression(Compression.Algorithm.NONE).build();
  DataBlockEncoder.EncodedSeeker seeker =
    encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
  seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
  Cell cell = seeker.getCell();
  Assert.assertEquals(expectedKV.getLength(), ((KeyValue) cell).getLength());
}
 

/**
 * Test that we don't cover other columns when we have a delete column.
 */
@Test
public void testDeleteColumnCorrectlyCoversColumns() {
  ApplyAndFilterDeletesFilter filter = new ApplyAndFilterDeletesFilter(EMPTY_SET);
  KeyValue d = createKvForType(Type.DeleteColumn, 12);
  byte[] qual2 = Bytes.add(qualifier, Bytes.toBytes("-other"));
  KeyValue put = new KeyValue(row, family, qual2, 11, Type.Put, value);

  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // different column put should still be visible
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));

  // set a delete family, but in the past
  d = createKvForType(Type.DeleteFamily, 10);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // add back in the original delete column
  d = createKvForType(Type.DeleteColumn, 11);
  assertEquals("Didn't filter out delete column", ReturnCode.SKIP, filter.filterKeyValue(d));
  // onto a different family, so that must be visible too
  assertEquals("Filtered out put with different column than the delete", ReturnCode.INCLUDE,
    filter.filterKeyValue(put));
}
 

/**
 * Copied from {@link TestKeyValue}
 * @throws Exception
 */
@Test
public void testColumnCompare() throws Exception {
  final byte [] a = Bytes.toBytes("aaa");
  byte [] family1 = Bytes.toBytes("abc");
  byte [] qualifier1 = Bytes.toBytes("def");
  byte [] family2 = Bytes.toBytes("abcd");
  byte [] qualifier2 = Bytes.toBytes("ef");

  KeyValue aaa = new ClientKeyValue(a, family1, qualifier1, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family2, qualifier2));
  assertTrue(aaa.matchingColumn(family1, qualifier1));
  aaa = new ClientKeyValue(a, family2, qualifier2, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family1, qualifier1));
  assertTrue(aaa.matchingColumn(family2,qualifier2));
  byte [] nullQualifier = new byte[0];
  aaa = new ClientKeyValue(a, family1, nullQualifier, 0L, Type.Put, a);
  assertTrue(aaa.matchingColumn(family1,null));
  assertFalse(aaa.matchingColumn(family2,qualifier2));
}
 

public static KeyValue newKeyValue(byte[] key, int keyOffset, int keyLength, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, keyOffset, keyLength,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");


  KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  kv.setSequenceId(0);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)kv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);
  //add the kvs from the mutation
  table.addPendingUpdates(m.get(fam, qual));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 

@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(new DelegateComparator(new CellComparatorImpl()){
      @Override
      public int compare(Cell leftCell, Cell rightCell) {
          return super.compare(leftCell, rightCell, true);
      }
  });
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 

public static KeyValue newKeyValue(ImmutableBytesWritable key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key.get(), key.getOffset(), key.getLength(),
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

/**
 * Test a corner case when the family qualifier is a prefix of the column qualifier.
 */
@Test
public void testColumnCompare_prefix() throws Exception {
  final byte[] a = Bytes.toBytes("aaa");
  byte[] family1 = Bytes.toBytes("abc");
  byte[] qualifier1 = Bytes.toBytes("def");
  byte[] family2 = Bytes.toBytes("ab");
  byte[] qualifier2 = Bytes.toBytes("def");

  KeyValue aaa = new ClientKeyValue(a, family1, qualifier1, 0L, Type.Put, a);
  assertFalse(aaa.matchingColumn(family2, qualifier2));
}
 

@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);


  Put pendingUpdate = new Put(row);
  pendingUpdate.addColumn(fam, qual, ts, val);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, pendingUpdate);


  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
}
 

public static KeyValue newKeyValue(byte[] key, int keyOffset, int keyLength, byte[] cf, 
    int cfOffset, int cfLength, byte[] cq, int cqOffset, int cqLength, long ts, byte[] value, 
    int valueOffset, int valueLength) {
    return new KeyValue(key, keyOffset, keyLength,
            cf, cfOffset, cfLength,
            cq, cqOffset, cqLength,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

public static KeyValue newKeyValue(byte[] key, int keyOffset, int keyLength, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key, keyOffset, keyLength,
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

public static KeyValue newKeyValue(ImmutableBytesWritable key, byte[] cf, byte[] cq, long ts, byte[] value, int valueOffset, int valueLength) {
    return new KeyValue(key.get(), key.getOffset(), key.getLength(),
            cf, 0, cf.length,
            cq, 0, cq.length,
            ts, Type.Put,
            value, valueOffset, valueLength);
}
 

public KeyValue create(Text key, byte[] value, int voffset, int vlen) {
    return new KeyValue(key.getBytes(), 0, key.getLength(), //
            cfBytes, 0, cfBytes.length, //
            qBytes, 0, qBytes.length, //
            timestamp, Type.Put, //
            value, voffset, vlen);
}
 

@Test(dataProvider = "cell-retain-options", timeOut = 60_000)
public void testShouldRetainNonTransactionallyDeletedCellMethod(int optionIdx, boolean retainOption)
        throws Exception {

    // Create required mocks
    @SuppressWarnings("unchecked")
    ObserverContext<RegionCoprocessorEnvironment> ctx = mock(ObserverContext.class);
    InternalScanner internalScanner = mock(InternalScanner.class);
    CommitTable.Client ctClient = mock(CommitTable.Client.class);

    RegionCoprocessorEnvironment rce = mock(RegionCoprocessorEnvironment.class);
    HRegion hRegion = mock(HRegion.class);
    HRegionInfo hRegionInfo = mock(HRegionInfo.class);
    SettableFuture<Long> f = SettableFuture.create();

    // Wire required mock internals
    f.set(TEST_TS);
    when(ctClient.readLowWatermark()).thenReturn(f);
    when(ctx.getEnvironment()).thenReturn(rce);
    when(rce.getRegion()).thenReturn(hRegion);
    when(hRegion.getRegionInfo()).thenReturn(hRegionInfo);

    LOG.info("Testing when retain is {}", retainOption);
    try (CompactorScanner scanner = spy(new CompactorScanner(ctx,
            internalScanner,
            ctClient,
            false,
            retainOption))) {

        // Different cell types to test
        KeyValue regularKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Put);
        KeyValue deleteKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.Delete);
        KeyValue deleteColumnKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteColumn);
        KeyValue deleteFamilyKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamily);
        KeyValue deleteFamilyVersionKV = new KeyValue(Bytes.toBytes("test-row"), TEST_TS, Type.DeleteFamilyVersion);

        assertFalse(scanner.shouldRetainNonTransactionallyDeletedCell(regularKV));
        assertEquals(scanner.shouldRetainNonTransactionallyDeletedCell(deleteKV), retainOption);
        assertEquals(scanner.shouldRetainNonTransactionallyDeletedCell(deleteColumnKV), retainOption);
        assertEquals(scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyKV), retainOption);
        assertEquals(scanner.shouldRetainNonTransactionallyDeletedCell(deleteFamilyVersionKV), retainOption);

    }

}
 

/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setMemstoreTS(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  // add the kvs from the mutation
  KeyValue kv = m.get(fam, qual).get(0);
  kv.setMemstoreTS(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 

@Override
public KeyValue buildPut(ImmutableBytesWritable row, ImmutableBytesWritable family,
    ImmutableBytesWritable qualifier, long ts, ImmutableBytesWritable value) {
  return new ClientKeyValue(row, family, qualifier, ts, Type.Put, value);
}
 

private byte[] getSomeKey(int rowId) {
  KeyValue kv = new KeyValue(Bytes.toBytes(String.format(localFormatter, Integer.valueOf(rowId))),
      Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put);
  return kv.getKey();
}
 

private static Cell getOffheapCell(byte [] row, byte [] family, byte [] qualifier) {
  KeyValue kvCell = new KeyValue(row, family, qualifier, 0L, Type.Put, row);
  ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length);
  ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length);
  return new ByteBufferKeyValue(buf, 0, buf.capacity(), 0L);
}