Java Code Examples for org.apache.hadoop.hbase.KeyValue#setSequenceId()

@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR);
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  KeyValueScanner scanner = store.getScanner();
  KeyValue first = KeyValue.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 

protected ImmutableSegment createTestImmutableSegment() {
  ImmutableSegment segment1 = SegmentFactory.instance().createImmutableSegment(this.comparator);
  final byte[] one = Bytes.toBytes(1);
  final byte[] two = Bytes.toBytes(2);
  final byte[] f = Bytes.toBytes(FAMILY);
  final byte[] q = Bytes.toBytes(COLUMN);
  final byte[] v = Bytes.toBytes(3);
  final KeyValue kv1 = new KeyValue(one, f, q, System.currentTimeMillis(), v);
  final KeyValue kv2 = new KeyValue(two, f, q, System.currentTimeMillis(), v);
  // the seqId of first cell less than Integer.MAX_VALUE,
  // the seqId of second cell greater than integer.MAX_VALUE
  kv1.setSequenceId(LESS_THAN_INTEGER_MAX_VALUE_SEQ_ID);
  kv2.setSequenceId(GREATER_THAN_INTEGER_MAX_VALUE_SEQ_ID);
  segment1.internalAdd(kv1, false, null, true);
  segment1.internalAdd(kv2, false, null, true);
  return segment1;
}
 

/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);

  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);

  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
}
 

static KeyValue readCell(PositionedByteRange pbr) throws Exception {
  int kvStartPos = pbr.getPosition();
  int keyLen = pbr.getInt();
  int valLen = pbr.getInt();
  pbr.setPosition(pbr.getPosition() + keyLen + valLen); // Skip the key and value section
  int tagsLen = ((pbr.get() & 0xff) << 8) ^ (pbr.get() & 0xff);
  pbr.setPosition(pbr.getPosition() + tagsLen); // Skip the tags section
  long mvcc = pbr.getVLong();
  KeyValue kv = new KeyValue(pbr.getBytes(), kvStartPos,
      (int) KeyValue.getKeyValueDataStructureSize(keyLen, valLen, tagsLen));
  kv.setSequenceId(mvcc);
  return kv;
}
 

@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);


  Put pendingUpdate = new Put(row);
  pendingUpdate.addColumn(fam, qual, ts, val);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, pendingUpdate);


  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
}
 

/**
 * Tests that the timeOfOldestEdit is updated correctly for the
 * various edit operations in memstore.
 */
@Override
@Test
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    long t = memstore.timeOfOldestEdit();
    assertEquals(Long.MAX_VALUE, t);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // The method will also assert
    // the value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore, true);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
   t = runSnapshot(memstore, true);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000, null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 

/**
 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
 * as older keyvalues are deleted from the memstore.
 *
 * @throws Exception
 */
@Override
@Test
public void testUpsertMemstoreSize() throws Exception {
  MemStoreSize oldSize = memstore.size();

  List<Cell> l = new ArrayList<>();
  KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
  KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
  KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");

  kv1.setSequenceId(1);
  kv2.setSequenceId(1);
  kv3.setSequenceId(1);
  l.add(kv1);
  l.add(kv2);
  l.add(kv3);

  this.memstore.upsert(l, 2, null);// readpoint is 2
  MemStoreSize newSize = this.memstore.size();
  assert (newSize.getDataSize() > oldSize.getDataSize());
  //The kv1 should be removed.
  assert (memstore.getActive().getCellsCount() == 2);

  KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
  kv4.setSequenceId(1);
  l.clear();
  l.add(kv4);
  this.memstore.upsert(l, 3, null);
  assertEquals(newSize, this.memstore.size());
  //The kv2 should be removed.
  assert (memstore.getActive().getCellsCount() == 2);
  //this.memstore = null;
}
 

/**
 * Tests that the timeOfOldestEdit is updated correctly for the
 * various edit operations in memstore.
 * @throws Exception
 */
@Test
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    DefaultMemStore memstore = new DefaultMemStore();
    long t = memstore.timeOfOldestEdit();
    assertEquals(Long.MAX_VALUE, t);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // snapshot() will reset timeOfOldestEdit. The method will also assert the
    // value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000, null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 

@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(new DelegateComparator(new CellComparatorImpl()){
      @Override
      public int compare(Cell leftCell, Cell rightCell) {
          return super.compare(leftCell, rightCell, true);
      }
  });
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 

@Test
public void testMemstoreConcurrentControl() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v = Bytes.toBytes("value");

  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv1 = new KeyValue(row, f, q1, v);
  kv1.setSequenceId(w.getWriteNumber());
  memstore.add(kv1, null);

  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{});

  mvcc.completeAndWait(w);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1});

  w = mvcc.begin();
  KeyValue kv2 = new KeyValue(row, f, q2, v);
  kv2.setSequenceId(w.getWriteNumber());
  memstore.add(kv2, null);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1});

  mvcc.completeAndWait(w);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1, kv2});
}
 

private Path writeToHFile(long l, String hFilePath, String pathStr, boolean nativeHFile)
    throws IOException {
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  final Path hfilePath = new Path(hFilePath);
  fs.mkdirs(hfilePath);
  Path path = new Path(pathStr);
  HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
  Assert.assertNotNull(wf);
  HFileContext context = new HFileContextBuilder().build();
  HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
  KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
      Bytes.toBytes("version2"));

  // Set cell seq id to test bulk load native hfiles.
  if (nativeHFile) {
    // Set a big seq id. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip
    // this kv.
    kv.setSequenceId(9999999);
  }

  writer.append(kv);

  if (nativeHFile) {
    // Set a big MAX_SEQ_ID_KEY. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip its
    // kv.
    writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(new Long(9999999)));
  }
  else {
  writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
  }
  writer.close();
  return hfilePath;
}
 

@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");


  KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  kv.setSequenceId(0);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)kv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);
  //add the kvs from the mutation
  table.addPendingUpdates(m.get(fam, qual));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 

/**
 * When we insert a higher-memstoreTS deletion of a cell but with
 * the same timestamp, we still need to provide consistent reads
 * for the same scanner.
 */
@Test
public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v1 = Bytes.toBytes("value1");
  // INSERT 1: Write both columns val1
  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv11 = new KeyValue(row, f, q1, v1);
  kv11.setSequenceId(w.getWriteNumber());
  memstore.add(kv11, null);

  KeyValue kv12 = new KeyValue(row, f, q2, v1);
  kv12.setSequenceId(w.getWriteNumber());
  memstore.add(kv12, null);
  mvcc.completeAndWait(w);

  // BEFORE STARTING INSERT 2, SEE FIRST KVS
  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // START DELETE: Insert delete for one of the columns
  w = mvcc.begin();
  KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
      KeyValue.Type.DeleteColumn);
  kvDel.setSequenceId(w.getWriteNumber());
  memstore.add(kvDel, null);

  // BEFORE COMPLETING DELETE, SEE FIRST KVS
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // COMPLETE DELETE
  mvcc.completeAndWait(w);

  // NOW WE SHOULD SEE DELETE
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
}
 

/**
 * Regression test for HBASE-2616, HBASE-2670.
 * When we insert a higher-memstoreTS version of a cell but with
 * the same timestamp, we still need to provide consistent reads
 * for the same scanner.
 */
@Test
public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v1 = Bytes.toBytes("value1");
  final byte[] v2 = Bytes.toBytes("value2");

  // INSERT 1: Write both columns val1
  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv11 = new KeyValue(row, f, q1, v1);
  kv11.setSequenceId(w.getWriteNumber());
  memstore.add(kv11, null);

  KeyValue kv12 = new KeyValue(row, f, q2, v1);
  kv12.setSequenceId(w.getWriteNumber());
  memstore.add(kv12, null);
  mvcc.completeAndWait(w);

  // BEFORE STARTING INSERT 2, SEE FIRST KVS
  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // START INSERT 2: Write both columns val2
  w = mvcc.begin();
  KeyValue kv21 = new KeyValue(row, f, q1, v2);
  kv21.setSequenceId(w.getWriteNumber());
  memstore.add(kv21, null);

  KeyValue kv22 = new KeyValue(row, f, q2, v2);
  kv22.setSequenceId(w.getWriteNumber());
  memstore.add(kv22, null);

  // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // COMPLETE INSERT 2
  mvcc.completeAndWait(w);

  // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
  // See HBASE-1485 for discussion about what we should do with
  // the duplicate-TS inserts
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
}
 

@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  Put pendingUpdate = new Put(row);
  pendingUpdate.add(fam, qual, ts, val);
  LocalTableState table = new LocalTableState(env, state, pendingUpdate);

  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 

private static KeyValue makeKV(int rowNum, int cqNum, byte[] familyName) {
  KeyValue kv = new KeyValue(ROWS[rowNum], familyName, QUALS[cqNum], TS,
      VALUES[rowNum % VALUESIZE]);
  kv.setSequenceId(makeMVCC(rowNum, cqNum));
  return kv;
}
 

static int writeTestKeyValues(HFileBlock.Writer hbw, int seed, boolean includesMemstoreTS,
    boolean useTag) throws IOException {
  List<KeyValue> keyValues = new ArrayList<>();
  Random randomizer = new Random(42L + seed); // just any fixed number

  // generate keyValues
  for (int i = 0; i < NUM_KEYVALUES; ++i) {
    byte[] row;
    long timestamp;
    byte[] family;
    byte[] qualifier;
    byte[] value;

    // generate it or repeat, it should compress well
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      row = CellUtil.cloneRow(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      row = new byte[FIELD_LENGTH];
      randomizer.nextBytes(row);
    }
    if (0 == i) {
      family = new byte[FIELD_LENGTH];
      randomizer.nextBytes(family);
    } else {
      family = CellUtil.cloneFamily(keyValues.get(0));
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      qualifier = CellUtil.cloneQualifier(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      qualifier = new byte[FIELD_LENGTH];
      randomizer.nextBytes(qualifier);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      value = CellUtil.cloneValue(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      value = new byte[FIELD_LENGTH];
      randomizer.nextBytes(value);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      timestamp = keyValues.get(
          randomizer.nextInt(keyValues.size())).getTimestamp();
    } else {
      timestamp = randomizer.nextLong();
    }
    if (!useTag) {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
    } else {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value,
          new Tag[] { new ArrayBackedTag((byte) 1, Bytes.toBytes("myTagVal")) }));
    }
  }

  // sort it and write to stream
  int totalSize = 0;
  Collections.sort(keyValues, CellComparatorImpl.COMPARATOR);

  for (KeyValue kv : keyValues) {
    totalSize += kv.getLength();
    if (includesMemstoreTS) {
      long memstoreTS = randomizer.nextLong();
      kv.setSequenceId(memstoreTS);
      totalSize += WritableUtils.getVIntSize(memstoreTS);
    }
    hbw.write(kv);
  }
  return totalSize;
}
 

/**
 * Provides access to compressed value.
 * @param headerSize header size of the block.
 * @return Forwards sequential iterator.
 */
public Iterator<Cell> getIterator(int headerSize) {
  final int rawSize = rawKVs.length;
  byte[] encodedDataWithHeader = getEncodedData();
  int bytesToSkip = headerSize + Bytes.SIZEOF_SHORT;
  ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
      bytesToSkip, encodedDataWithHeader.length - bytesToSkip);
  final DataInputStream dis = new DataInputStream(bais);

  return new Iterator<Cell>() {
    private ByteBuffer decompressedData = null;
    private Iterator<Boolean> it = isTagsLenZero.iterator();

    @Override
    public boolean hasNext() {
      if (decompressedData == null) {
        return rawSize > 0;
      }
      return decompressedData.hasRemaining();
    }

    @Override
    public Cell next() {
      if (decompressedData == null) {
        try {
          decompressedData = dataBlockEncoder.decodeKeyValues(dis, dataBlockEncoder
              .newDataBlockDecodingContext(meta));
        } catch (IOException e) {
          throw new RuntimeException("Problem with data block encoder, " +
              "most likely it requested more bytes than are available.", e);
        }
        decompressedData.rewind();
      }
      int offset = decompressedData.position();
      int klen = decompressedData.getInt();
      int vlen = decompressedData.getInt();
      int tagsLen = 0;
      ByteBufferUtils.skip(decompressedData, klen + vlen);
      // Read the tag length in case when stream contain tags
      if (meta.isIncludesTags()) {
        boolean noTags = true;
        if (it.hasNext()) {
          noTags = it.next();
        }
        // ROW_INDEX_V1 will not put tagsLen back in cell if it is zero, there is no need
        // to read short here.
        if (!(encoding.equals(DataBlockEncoding.ROW_INDEX_V1) && noTags)) {
          tagsLen = ((decompressedData.get() & 0xff) << 8) ^ (decompressedData.get() & 0xff);
          ByteBufferUtils.skip(decompressedData, tagsLen);
        }
      }
      KeyValue kv = new KeyValue(decompressedData.array(), offset,
          (int) KeyValue.getKeyValueDataStructureSize(klen, vlen, tagsLen));
      if (meta.isIncludesMvcc()) {
        long mvccVersion = ByteBufferUtils.readVLong(decompressedData);
        kv.setSequenceId(mvccVersion);
      }
      return kv;
    }

    @Override
    public void remove() {
      throw new NotImplementedException("remove() is not supported!");
    }

    @Override
    public String toString() {
      return "Iterator of: " + dataBlockEncoder.getClass().getName();
    }

  };
}
 

/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 

static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {

        kv.setSequenceId(sequenceId);

    }