Java Code Examples for org.apache.hadoop.hbase.KeyValue#setSequenceId()

The following examples show how to use org.apache.hadoop.hbase.KeyValue#setSequenceId() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix   File: TestIndexMemStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(IndexMemStore.COMPARATOR);
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  KeyValueScanner scanner = store.getScanner();
  KeyValue first = KeyValue.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 
Example 2
protected ImmutableSegment createTestImmutableSegment() {
  ImmutableSegment segment1 = SegmentFactory.instance().createImmutableSegment(this.comparator);
  final byte[] one = Bytes.toBytes(1);
  final byte[] two = Bytes.toBytes(2);
  final byte[] f = Bytes.toBytes(FAMILY);
  final byte[] q = Bytes.toBytes(COLUMN);
  final byte[] v = Bytes.toBytes(3);
  final KeyValue kv1 = new KeyValue(one, f, q, System.currentTimeMillis(), v);
  final KeyValue kv2 = new KeyValue(two, f, q, System.currentTimeMillis(), v);
  // the seqId of first cell less than Integer.MAX_VALUE,
  // the seqId of second cell greater than integer.MAX_VALUE
  kv1.setSequenceId(LESS_THAN_INTEGER_MAX_VALUE_SEQ_ID);
  kv2.setSequenceId(GREATER_THAN_INTEGER_MAX_VALUE_SEQ_ID);
  segment1.internalAdd(kv1, false, null, true);
  segment1.internalAdd(kv2, false, null, true);
  return segment1;
}
 
Example 3
Source Project: phoenix   File: LocalTableStateTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testCorrectOrderingWithLazyLoadingColumns() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  Configuration conf = new Configuration(false);
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);
  Mockito.when(env.getConfiguration()).thenReturn(conf);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");


  KeyValue kv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  kv.setSequenceId(0);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)kv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);
  //add the kvs from the mutation
  table.addPendingUpdates(m.get(fam, qual));

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  //check that our value still shows up first on scan, even though this is a lazy load
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", m.get(fam, qual).get(0), s.next());
}
 
Example 4
Source Project: hbase   File: TestScannerWithBulkload.java    License: Apache License 2.0 5 votes vote down vote up
private Path writeToHFile(long l, String hFilePath, String pathStr, boolean nativeHFile)
    throws IOException {
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  final Path hfilePath = new Path(hFilePath);
  fs.mkdirs(hfilePath);
  Path path = new Path(pathStr);
  HFile.WriterFactory wf = HFile.getWriterFactoryNoCache(TEST_UTIL.getConfiguration());
  Assert.assertNotNull(wf);
  HFileContext context = new HFileContextBuilder().build();
  HFile.Writer writer = wf.withPath(fs, path).withFileContext(context).create();
  KeyValue kv = new KeyValue(Bytes.toBytes("row1"), Bytes.toBytes("col"), Bytes.toBytes("q"), l,
      Bytes.toBytes("version2"));

  // Set cell seq id to test bulk load native hfiles.
  if (nativeHFile) {
    // Set a big seq id. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip
    // this kv.
    kv.setSequenceId(9999999);
  }

  writer.append(kv);

  if (nativeHFile) {
    // Set a big MAX_SEQ_ID_KEY. Scan should not look at this seq id in a bulk loaded file.
    // Scan should only look at the seq id appended at the bulk load time, and not skip its
    // kv.
    writer.appendFileInfo(MAX_SEQ_ID_KEY, Bytes.toBytes(new Long(9999999)));
  }
  else {
  writer.appendFileInfo(BULKLOAD_TIME_KEY, Bytes.toBytes(System.currentTimeMillis()));
  }
  writer.close();
  return hfilePath;
}
 
Example 5
Source Project: hbase   File: TestDefaultMemStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMemstoreConcurrentControl() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v = Bytes.toBytes("value");

  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv1 = new KeyValue(row, f, q1, v);
  kv1.setSequenceId(w.getWriteNumber());
  memstore.add(kv1, null);

  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{});

  mvcc.completeAndWait(w);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1});

  w = mvcc.begin();
  KeyValue kv2 = new KeyValue(row, f, q2, v);
  kv2.setSequenceId(w.getWriteNumber());
  memstore.add(kv2, null);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1});

  mvcc.completeAndWait(w);

  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv1, kv2});
}
 
Example 6
Source Project: phoenix   File: TestIndexMemStore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(new DelegateComparator(new CellComparatorImpl()){
      @Override
      public int compare(Cell leftCell, Cell rightCell) {
          return super.compare(leftCell, rightCell, true);
      }
  });
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 
Example 7
Source Project: phoenix   File: LocalTableStateTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.addColumn(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);

  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, m);

  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
}
 
Example 8
Source Project: hbase   File: TestDefaultMemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the timeOfOldestEdit is updated correctly for the
 * various edit operations in memstore.
 * @throws Exception
 */
@Test
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    DefaultMemStore memstore = new DefaultMemStore();
    long t = memstore.timeOfOldestEdit();
    assertEquals(Long.MAX_VALUE, t);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // snapshot() will reset timeOfOldestEdit. The method will also assert the
    // value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    t = runSnapshot(memstore);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000, null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example 9
Source Project: hbase   File: TestCompactingMemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
 * as older keyvalues are deleted from the memstore.
 *
 * @throws Exception
 */
@Override
@Test
public void testUpsertMemstoreSize() throws Exception {
  MemStoreSize oldSize = memstore.size();

  List<Cell> l = new ArrayList<>();
  KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
  KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
  KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");

  kv1.setSequenceId(1);
  kv2.setSequenceId(1);
  kv3.setSequenceId(1);
  l.add(kv1);
  l.add(kv2);
  l.add(kv3);

  this.memstore.upsert(l, 2, null);// readpoint is 2
  MemStoreSize newSize = this.memstore.size();
  assert (newSize.getDataSize() > oldSize.getDataSize());
  //The kv1 should be removed.
  assert (memstore.getActive().getCellsCount() == 2);

  KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
  kv4.setSequenceId(1);
  l.clear();
  l.add(kv4);
  this.memstore.upsert(l, 3, null);
  assertEquals(newSize, this.memstore.size());
  //The kv2 should be removed.
  assert (memstore.getActive().getCellsCount() == 2);
  //this.memstore = null;
}
 
Example 10
Source Project: hbase   File: TestCompactingMemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the timeOfOldestEdit is updated correctly for the
 * various edit operations in memstore.
 */
@Override
@Test
public void testUpdateToTimeOfOldestEdit() throws Exception {
  try {
    EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest();
    EnvironmentEdgeManager.injectEdge(edge);
    long t = memstore.timeOfOldestEdit();
    assertEquals(Long.MAX_VALUE, t);

    // test the case that the timeOfOldestEdit is updated after a KV add
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
    // The method will also assert
    // the value is reset to Long.MAX_VALUE
    t = runSnapshot(memstore, true);

    // test the case that the timeOfOldestEdit is updated after a KV delete
    memstore.add(KeyValueTestUtil.create("r", "f", "q", 100, KeyValue.Type.Delete, "v"), null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
   t = runSnapshot(memstore, true);

    // test the case that the timeOfOldestEdit is updated after a KV upsert
    List<Cell> l = new ArrayList<>();
    KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
    kv1.setSequenceId(100);
    l.add(kv1);
    memstore.upsert(l, 1000, null);
    t = memstore.timeOfOldestEdit();
    assertTrue(t == 1234);
  } finally {
    EnvironmentEdgeManager.reset();
  }
}
 
Example 11
Source Project: phoenix   File: LocalTableStateTest.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  Region region = Mockito.mock(Region.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);


  Put pendingUpdate = new Put(row);
  pendingUpdate.addColumn(fam, qual, ts, val);
  HashMap<ImmutableBytesPtr, List<Cell>> rowKeyPtrToCells =
          new  HashMap<ImmutableBytesPtr, List<Cell>>();
  rowKeyPtrToCells.put(new ImmutableBytesPtr(row), Collections.singletonList((Cell)storedKv));
  CachedLocalTable cachedLocalTable = CachedLocalTable.build(rowKeyPtrToCells);
  LocalTableState table = new LocalTableState(cachedLocalTable, pendingUpdate);


  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<CoveredDeleteScanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col), false, false, indexMetaData);
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
}
 
Example 12
static KeyValue readCell(PositionedByteRange pbr) throws Exception {
  int kvStartPos = pbr.getPosition();
  int keyLen = pbr.getInt();
  int valLen = pbr.getInt();
  pbr.setPosition(pbr.getPosition() + keyLen + valLen); // Skip the key and value section
  int tagsLen = ((pbr.get() & 0xff) << 8) ^ (pbr.get() & 0xff);
  pbr.setPosition(pbr.getPosition() + tagsLen); // Skip the tags section
  long mvcc = pbr.getVLong();
  KeyValue kv = new KeyValue(pbr.getBytes(), kvStartPos,
      (int) KeyValue.getKeyValueDataStructureSize(keyLen, valLen, tagsLen));
  kv.setSequenceId(mvcc);
  return kv;
}
 
Example 13
Source Project: phoenix   File: TestLocalTableState.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Test that we correctly rollback the state of keyvalue
 * @throws Exception
 */
@Test
@SuppressWarnings("unchecked")
public void testCorrectRollback() throws Exception {
  Put m = new Put(row);
  m.add(fam, qual, ts, val);
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final byte[] stored = Bytes.toBytes("stored-value");
  final KeyValue storedKv = new KeyValue(row, fam, qual, ts, Type.Put, stored);
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  LocalTableState table = new LocalTableState(env, state, m);
  // add the kvs from the mutation
  KeyValue kv = KeyValueUtil.ensureKeyValue(m.get(fam, qual).get(0));
  kv.setSequenceId(0);
  table.addPendingUpdates(kv);

  // setup the lookup
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  assertEquals("Didn't get the pending mutation's value first", kv, s.next());

  // rollback that value
  table.rollback(Arrays.asList(kv));
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Didn't correctly rollback the row - still found it!", null, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 
Example 14
Source Project: phoenix   File: TestLocalTableState.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("unchecked")
@Test
public void testOnlyLoadsRequestedColumns() throws Exception {
  // setup mocks
  RegionCoprocessorEnvironment env = Mockito.mock(RegionCoprocessorEnvironment.class);

  HRegion region = Mockito.mock(HRegion.class);
  Mockito.when(env.getRegion()).thenReturn(region);
  RegionScanner scanner = Mockito.mock(RegionScanner.class);
  Mockito.when(region.getScanner(Mockito.any(Scan.class))).thenReturn(scanner);
  final KeyValue storedKv =
      new KeyValue(row, fam, qual, ts, Type.Put, Bytes.toBytes("stored-value"));
  storedKv.setSequenceId(2);
  Mockito.when(scanner.next(Mockito.any(List.class))).thenAnswer(new Answer<Boolean>() {
    @Override
    public Boolean answer(InvocationOnMock invocation) throws Throwable {
      List<KeyValue> list = (List<KeyValue>) invocation.getArguments()[0];

      list.add(storedKv);
      return false;
    }
  });
  LocalHBaseState state = new LocalTable(env);
  Put pendingUpdate = new Put(row);
  pendingUpdate.add(fam, qual, ts, val);
  LocalTableState table = new LocalTableState(env, state, pendingUpdate);

  // do the lookup for the given column
  ColumnReference col = new ColumnReference(fam, qual);
  table.setCurrentTimestamp(ts);
  // check that the value is there
  Pair<Scanner, IndexUpdate> p = table.getIndexedColumnsTableState(Arrays.asList(col));
  Scanner s = p.getFirst();
  // make sure it read the table the one time
  assertEquals("Didn't get the stored keyvalue!", storedKv, s.next());

  // on the second lookup it shouldn't access the underlying table again - the cached columns
  // should know they are done
  p = table.getIndexedColumnsTableState(Arrays.asList(col));
  s = p.getFirst();
  assertEquals("Lost already loaded update!", storedKv, s.next());
  Mockito.verify(env, Mockito.times(1)).getRegion();
  Mockito.verify(region, Mockito.times(1)).getScanner(Mockito.any(Scan.class));
}
 
Example 15
Source Project: hbase   File: TestDefaultMemStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Regression test for HBASE-2616, HBASE-2670.
 * When we insert a higher-memstoreTS version of a cell but with
 * the same timestamp, we still need to provide consistent reads
 * for the same scanner.
 */
@Test
public void testMemstoreEditsVisibilityWithSameKey() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v1 = Bytes.toBytes("value1");
  final byte[] v2 = Bytes.toBytes("value2");

  // INSERT 1: Write both columns val1
  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv11 = new KeyValue(row, f, q1, v1);
  kv11.setSequenceId(w.getWriteNumber());
  memstore.add(kv11, null);

  KeyValue kv12 = new KeyValue(row, f, q2, v1);
  kv12.setSequenceId(w.getWriteNumber());
  memstore.add(kv12, null);
  mvcc.completeAndWait(w);

  // BEFORE STARTING INSERT 2, SEE FIRST KVS
  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // START INSERT 2: Write both columns val2
  w = mvcc.begin();
  KeyValue kv21 = new KeyValue(row, f, q1, v2);
  kv21.setSequenceId(w.getWriteNumber());
  memstore.add(kv21, null);

  KeyValue kv22 = new KeyValue(row, f, q2, v2);
  kv22.setSequenceId(w.getWriteNumber());
  memstore.add(kv22, null);

  // BEFORE COMPLETING INSERT 2, SEE FIRST KVS
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // COMPLETE INSERT 2
  mvcc.completeAndWait(w);

  // NOW SHOULD SEE NEW KVS IN ADDITION TO OLD KVS.
  // See HBASE-1485 for discussion about what we should do with
  // the duplicate-TS inserts
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv21, kv11, kv22, kv12});
}
 
Example 16
Source Project: hbase   File: TestDefaultMemStore.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * When we insert a higher-memstoreTS deletion of a cell but with
 * the same timestamp, we still need to provide consistent reads
 * for the same scanner.
 */
@Test
public void testMemstoreDeletesVisibilityWithSameKey() throws IOException {
  final byte[] row = Bytes.toBytes(1);
  final byte[] f = Bytes.toBytes("family");
  final byte[] q1 = Bytes.toBytes("q1");
  final byte[] q2 = Bytes.toBytes("q2");
  final byte[] v1 = Bytes.toBytes("value1");
  // INSERT 1: Write both columns val1
  MultiVersionConcurrencyControl.WriteEntry w =
      mvcc.begin();

  KeyValue kv11 = new KeyValue(row, f, q1, v1);
  kv11.setSequenceId(w.getWriteNumber());
  memstore.add(kv11, null);

  KeyValue kv12 = new KeyValue(row, f, q2, v1);
  kv12.setSequenceId(w.getWriteNumber());
  memstore.add(kv12, null);
  mvcc.completeAndWait(w);

  // BEFORE STARTING INSERT 2, SEE FIRST KVS
  KeyValueScanner s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // START DELETE: Insert delete for one of the columns
  w = mvcc.begin();
  KeyValue kvDel = new KeyValue(row, f, q2, kv11.getTimestamp(),
      KeyValue.Type.DeleteColumn);
  kvDel.setSequenceId(w.getWriteNumber());
  memstore.add(kvDel, null);

  // BEFORE COMPLETING DELETE, SEE FIRST KVS
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kv12});

  // COMPLETE DELETE
  mvcc.completeAndWait(w);

  // NOW WE SHOULD SEE DELETE
  s = this.memstore.getScanners(mvcc.getReadPoint()).get(0);
  assertScannerResults(s, new KeyValue[]{kv11, kvDel, kv12});
}
 
Example 17
Source Project: hbase   File: TestReversibleScanners.java    License: Apache License 2.0 4 votes vote down vote up
private static KeyValue makeKV(int rowNum, int cqNum, byte[] familyName) {
  KeyValue kv = new KeyValue(ROWS[rowNum], familyName, QUALS[cqNum], TS,
      VALUES[rowNum % VALUESIZE]);
  kv.setSequenceId(makeMVCC(rowNum, cqNum));
  return kv;
}
 
Example 18
Source Project: hbase   File: TestHFileBlock.java    License: Apache License 2.0 4 votes vote down vote up
static int writeTestKeyValues(HFileBlock.Writer hbw, int seed, boolean includesMemstoreTS,
    boolean useTag) throws IOException {
  List<KeyValue> keyValues = new ArrayList<>();
  Random randomizer = new Random(42L + seed); // just any fixed number

  // generate keyValues
  for (int i = 0; i < NUM_KEYVALUES; ++i) {
    byte[] row;
    long timestamp;
    byte[] family;
    byte[] qualifier;
    byte[] value;

    // generate it or repeat, it should compress well
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      row = CellUtil.cloneRow(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      row = new byte[FIELD_LENGTH];
      randomizer.nextBytes(row);
    }
    if (0 == i) {
      family = new byte[FIELD_LENGTH];
      randomizer.nextBytes(family);
    } else {
      family = CellUtil.cloneFamily(keyValues.get(0));
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      qualifier = CellUtil.cloneQualifier(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      qualifier = new byte[FIELD_LENGTH];
      randomizer.nextBytes(qualifier);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      value = CellUtil.cloneValue(keyValues.get(randomizer.nextInt(keyValues.size())));
    } else {
      value = new byte[FIELD_LENGTH];
      randomizer.nextBytes(value);
    }
    if (0 < i && randomizer.nextFloat() < CHANCE_TO_REPEAT) {
      timestamp = keyValues.get(
          randomizer.nextInt(keyValues.size())).getTimestamp();
    } else {
      timestamp = randomizer.nextLong();
    }
    if (!useTag) {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value));
    } else {
      keyValues.add(new KeyValue(row, family, qualifier, timestamp, value,
          new Tag[] { new ArrayBackedTag((byte) 1, Bytes.toBytes("myTagVal")) }));
    }
  }

  // sort it and write to stream
  int totalSize = 0;
  Collections.sort(keyValues, CellComparatorImpl.COMPARATOR);

  for (KeyValue kv : keyValues) {
    totalSize += kv.getLength();
    if (includesMemstoreTS) {
      long memstoreTS = randomizer.nextLong();
      kv.setSequenceId(memstoreTS);
      totalSize += WritableUtils.getVIntSize(memstoreTS);
    }
    hbw.write(kv);
  }
  return totalSize;
}
 
Example 19
Source Project: hbase   File: EncodedDataBlock.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Provides access to compressed value.
 * @param headerSize header size of the block.
 * @return Forwards sequential iterator.
 */
public Iterator<Cell> getIterator(int headerSize) {
  final int rawSize = rawKVs.length;
  byte[] encodedDataWithHeader = getEncodedData();
  int bytesToSkip = headerSize + Bytes.SIZEOF_SHORT;
  ByteArrayInputStream bais = new ByteArrayInputStream(encodedDataWithHeader,
      bytesToSkip, encodedDataWithHeader.length - bytesToSkip);
  final DataInputStream dis = new DataInputStream(bais);

  return new Iterator<Cell>() {
    private ByteBuffer decompressedData = null;
    private Iterator<Boolean> it = isTagsLenZero.iterator();

    @Override
    public boolean hasNext() {
      if (decompressedData == null) {
        return rawSize > 0;
      }
      return decompressedData.hasRemaining();
    }

    @Override
    public Cell next() {
      if (decompressedData == null) {
        try {
          decompressedData = dataBlockEncoder.decodeKeyValues(dis, dataBlockEncoder
              .newDataBlockDecodingContext(meta));
        } catch (IOException e) {
          throw new RuntimeException("Problem with data block encoder, " +
              "most likely it requested more bytes than are available.", e);
        }
        decompressedData.rewind();
      }
      int offset = decompressedData.position();
      int klen = decompressedData.getInt();
      int vlen = decompressedData.getInt();
      int tagsLen = 0;
      ByteBufferUtils.skip(decompressedData, klen + vlen);
      // Read the tag length in case when stream contain tags
      if (meta.isIncludesTags()) {
        boolean noTags = true;
        if (it.hasNext()) {
          noTags = it.next();
        }
        // ROW_INDEX_V1 will not put tagsLen back in cell if it is zero, there is no need
        // to read short here.
        if (!(encoding.equals(DataBlockEncoding.ROW_INDEX_V1) && noTags)) {
          tagsLen = ((decompressedData.get() & 0xff) << 8) ^ (decompressedData.get() & 0xff);
          ByteBufferUtils.skip(decompressedData, tagsLen);
        }
      }
      KeyValue kv = new KeyValue(decompressedData.array(), offset,
          (int) KeyValue.getKeyValueDataStructureSize(klen, vlen, tagsLen));
      if (meta.isIncludesMvcc()) {
        long mvccVersion = ByteBufferUtils.readVLong(decompressedData);
        kv.setSequenceId(mvccVersion);
      }
      return kv;
    }

    @Override
    public void remove() {
      throw new NotImplementedException("remove() is not supported!");
    }

    @Override
    public String toString() {
      return "Iterator of: " + dataBlockEncoder.getClass().getName();
    }

  };
}
 
Example 20
Source Project: phoenix-omid   File: HBaseShims.java    License: Apache License 2.0 2 votes vote down vote up
static public void setKeyValueSequenceId(KeyValue kv, int sequenceId) {

        kv.setSequenceId(sequenceId);

    }