Java Code Examples for org.apache.hadoop.hbase.CellComparatorImpl

The following examples show how to use org.apache.hadoop.hbase.CellComparatorImpl. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase   Source File: FixedFileTrailer.java    License: Apache License 2.0 6 votes vote down vote up
static CellComparator createComparator(String comparatorClassName) throws IOException {
  if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) {
    return CellComparatorImpl.COMPARATOR;
  } else if (comparatorClassName.equals(
    CellComparatorImpl.META_COMPARATOR.getClass().getName())) {
    return CellComparatorImpl.META_COMPARATOR;
  }
  try {
    Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
    if (comparatorClass != null) {
      return comparatorClass.getDeclaredConstructor().newInstance();
    }
    LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null.");
    return null;
  } catch (Exception e) {
    throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e);
  }
}
 
Example 2
Source Project: hbase   Source File: TestReplicationWALEntryFilters.java    License: Apache License 2.0 6 votes vote down vote up
private void assertEquals(Entry e1, Entry e2) {
  Assert.assertEquals(e1 == null, e2 == null);
  if (e1 == null) {
    return;
  }

  // do not compare WALKeys

  // compare kvs
  Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
  if (e1.getEdit() == null) {
    return;
  }
  List<Cell> cells1 = e1.getEdit().getCells();
  List<Cell> cells2 = e2.getEdit().getCells();
  Assert.assertEquals(cells1.size(), cells2.size());
  for (int i = 0; i < cells1.size(); i++) {
    CellComparatorImpl.COMPARATOR.compare(cells1.get(i), cells2.get(i));
  }
}
 
Example 3
Source Project: hbase   Source File: TestDefaultMemStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleVersionsSimple() throws Exception {
  DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR);
  byte [] row = Bytes.toBytes("testRow");
  byte [] family = Bytes.toBytes("testFamily");
  byte [] qf = Bytes.toBytes("testQualifier");
  long [] stamps = {1,2,3};
  byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
      Bytes.toBytes("value2")};
  KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
  KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
  KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);

  m.add(key0, null);
  m.add(key1, null);
  m.add(key2, null);

  assertTrue("Expected memstore to hold 3 values, actually has " +
      m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3);
}
 
Example 4
Source Project: hbase   Source File: TestKeyValueHeap.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSorted() throws IOException {
  // Cases that need to be checked are:
  // 1. The "smallest" Cell is in the same scanners as current
  // 2. Current scanner gets empty

  List<Cell> expected =
    Arrays.asList(kv111, kv112, kv113, kv114, kv115, kv121, kv122, kv211, kv212, kv213);

  List<Cell> actual = assertCells(expected, scanners);

  // Check if result is sorted according to Comparator
  for (int i = 0; i < actual.size() - 1; i++) {
    int ret = CellComparatorImpl.COMPARATOR.compare(actual.get(i), actual.get(i + 1));
    assertTrue(ret < 0);
  }
}
 
Example 5
Source Project: hbase   Source File: TestKeyValueHeap.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSeek() throws IOException {
  // Cases:
  // 1. Seek Cell that is not in scanner
  // 2. Check that smallest that is returned from a seek is correct
  List<Cell> expected = Arrays.asList(kv211);

  // Creating KeyValueHeap
  try (KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparatorImpl.COMPARATOR)) {
    Cell seekKv = new KeyValue(row2, fam1, null, null);
    kvh.seek(seekKv);

    List<Cell> actual = Arrays.asList(kvh.peek());

    assertEquals("Expected = " + Arrays.toString(expected.toArray()) + "\n Actual = " +
      Arrays.toString(actual.toArray()), expected, actual);
  }
}
 
Example 6
Source Project: hbase   Source File: TestScanWildcardColumnTracker.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void DisabledTestCheckColumnWrongOrder() {
  ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(
      0, VERSIONS, Long.MIN_VALUE, CellComparatorImpl.COMPARATOR);

  // Create list of qualifiers
  List<byte[]> qualifiers = new ArrayList<>(2);
  qualifiers.add(Bytes.toBytes("qualifier2"));
  qualifiers.add(Bytes.toBytes("qualifier1"));

  try {
    for (byte[] qualifier : qualifiers) {
      ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
        KeyValue.Type.Put.getCode(), false);
    }
    fail();
  } catch (IOException e) {
    // expected
  }
}
 
Example 7
Source Project: hbase   Source File: TestStripeCompactionPolicy.java    License: Apache License 2.0 6 votes vote down vote up
/** Verify arbitrary flush. */
protected void verifyFlush(StripeCompactionPolicy policy, StripeInformationProvider si,
    KeyValue[] input, KeyValue[][] expected, byte[][] boundaries) throws IOException {
  StoreFileWritersCapture writers = new StoreFileWritersCapture();
  StripeStoreFlusher.StripeFlushRequest req =
    policy.selectFlush(CellComparatorImpl.COMPARATOR, si, input.length);
  StripeMultiFileWriter mw = req.createWriter();
  mw.init(null, writers);
  for (KeyValue kv : input) {
    mw.append(kv);
  }
  boolean hasMetadata = boundaries != null;
  mw.commitWriters(0, false);
  writers.verifyKvs(expected, true, hasMetadata);
  if (hasMetadata) {
    writers.verifyBoundaries(boundaries);
  }
}
 
Example 8
Source Project: hbase   Source File: TestCompactingToCellFlatMapMemStore.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCountOfCellsAfterFlatteningByIterator() throws IOException {
  String[] keys1 = { "A", "B", "C" }; // A, B, C
  addRowsByKeysWith50Cols(memstore, keys1);
  // this should only flatten as there are no duplicates
  ((CompactingMemStore) memstore).flushInMemory();
  while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
    Threads.sleep(10);
  }
  // Just doing the cnt operation here
  MemStoreSegmentsIterator itr = new MemStoreMergerSegmentsIterator(
      ((CompactingMemStore) memstore).getImmutableSegments().getStoreSegments(),
      CellComparatorImpl.COMPARATOR, 10);
  int cnt = 0;
  try {
    while (itr.next() != null) {
      cnt++;
    }
  } finally {
    itr.close();
  }
  assertEquals("the count should be ", 150, cnt);
}
 
Example 9
Source Project: hbase   Source File: TestDefaultStoreEngine.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomParts() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName());
  conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
      DummyCompactionPolicy.class.getName());
  conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
      DummyStoreFlusher.class.getName());
  HStore mockStore = Mockito.mock(HStore.class);
  Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO);
  StoreEngine<?, ?, ?, ?> se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR);
  Assert.assertTrue(se instanceof DefaultStoreEngine);
  Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy);
  Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher);
  Assert.assertTrue(se.getCompactor() instanceof DummyCompactor);
}
 
Example 10
Source Project: hbase   Source File: TestDependentColumnFilter.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This shouldn't be confused with TestFilter#verifyScan
 * as expectedKeys is not the per row total, but the scan total
 *
 * @param s
 * @param expectedRows
 * @param expectedCells
 * @throws IOException
 */
private void verifyScan(Scan s, long expectedRows, long expectedCells)
throws IOException {
  InternalScanner scanner = this.region.getScanner(s);
  List<Cell> results = new ArrayList<>();
  int i = 0;
  int cells = 0;
  for (boolean done = true; done; i++) {
    done = scanner.next(results);
    Arrays.sort(results.toArray(new Cell[results.size()]),
        CellComparatorImpl.COMPARATOR);
    LOG.info("counter=" + i + ", " + results);
    if (results.isEmpty()) break;
    cells += results.size();
    assertTrue("Scanned too many rows! Only expected " + expectedRows +
        " total but already scanned " + (i+1), expectedRows > i);
    assertTrue("Expected " + expectedCells + " cells total but " +
        "already scanned " + cells, expectedCells >= cells);
    results.clear();
  }
  assertEquals("Expected " + expectedRows + " rows but scanned " + i +
      " rows", expectedRows, i);
  assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
          " cells", expectedCells, cells);
}
 
Example 11
Source Project: hbase   Source File: TestHFile.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testShortMidpointSameQual() {
  Cell left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(Bytes.toBytes("a"))
    .setFamily(Bytes.toBytes("a"))
    .setQualifier(Bytes.toBytes("a"))
    .setTimestamp(11)
    .setType(Type.Maximum.getCode())
    .setValue(HConstants.EMPTY_BYTE_ARRAY)
    .build();
  Cell right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(Bytes.toBytes("a"))
    .setFamily(Bytes.toBytes("a"))
    .setQualifier(Bytes.toBytes("a"))
    .setTimestamp(9)
    .setType(Type.Maximum.getCode())
    .setValue(HConstants.EMPTY_BYTE_ARRAY)
    .build();
  Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
  assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
  assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
}
 
Example 12
Source Project: hbase   Source File: TestDataBlockEncoders.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Test whether the decompression of first key is implemented correctly.
 * @throws IOException
 */
@Test
public void testFirstKeyInBlockOnSample() throws IOException {
  List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags);

  for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
    if (encoding.getEncoder() == null) {
      continue;
    }
    DataBlockEncoder encoder = encoding.getEncoder();
    ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv,
        getEncodingContext(Compression.Algorithm.NONE, encoding), this.useOffheapData);
    Cell key = encoder.getFirstKeyCellInBlock(new SingleByteBuff(encodedBuffer));
    KeyValue firstKv = sampleKv.get(0);
    if (0 != PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, key, firstKv)) {
      int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(key, firstKv, false, true);
      fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix));
    }
  }
}
 
Example 13
Source Project: hbase   Source File: TestProtobufUtil.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testToCell() {
  KeyValue kv1 =
      new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv2 =
      new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv3 =
      new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
  System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
  System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
  System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(),
    kv3.getLength());
  ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
  dbb.put(arr);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
  CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV);
  Cell newOffheapKV =
      ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell);
  assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
}
 
Example 14
Source Project: hbase   Source File: HFileContext.java    License: Apache License 2.0 6 votes vote down vote up
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
             Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
             int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
             Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
             byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
  this.usesHBaseChecksum = useHBaseChecksum;
  this.includesMvcc =  includesMvcc;
  this.includesTags = includesTags;
  this.compressAlgo = compressAlgo;
  this.compressTags = compressTags;
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
  this.blocksize = blockSize;
  if (encoding != null) {
    this.encoding = encoding;
  }
  this.cryptoContext = cryptoContext;
  this.fileCreateTime = fileCreateTime;
  this.hfileName = hfileName;
  this.columnFamily = columnFamily;
  this.tableName = tableName;
  // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
  // be the meta table comparator. Comparators are per table.
  this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
    CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
 
Example 15
Source Project: hbase   Source File: BoundedRecoveredHFilesOutputSink.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @return Returns a base HFile without compressions or encodings; good enough for recovery
 *   given hfile has metadata on how it was written.
 */
private StoreFileWriter createRecoveredHFileWriter(TableName tableName, String regionName,
    long seqId, String familyName, boolean isMetaTable) throws IOException {
  Path outputDir = WALSplitUtil.tryCreateRecoveredHFilesDir(walSplitter.rootFS, walSplitter.conf,
    tableName, regionName, familyName);
  StoreFileWriter.Builder writerBuilder =
      new StoreFileWriter.Builder(walSplitter.conf, CacheConfig.DISABLED, walSplitter.rootFS)
          .withOutputDir(outputDir);
  HFileContext hFileContext = new HFileContextBuilder().
    withChecksumType(HStore.getChecksumType(walSplitter.conf)).
    withBytesPerCheckSum(HStore.getBytesPerChecksum(walSplitter.conf)).
    withCellComparator(isMetaTable?
      CellComparatorImpl.META_COMPARATOR: CellComparatorImpl.COMPARATOR).build();
  return writerBuilder.withFileContext(hFileContext).build();
}
 
Example 16
Source Project: hbase   Source File: BloomFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new general (Row or RowCol) Bloom filter at the time of
 * {@link org.apache.hadoop.hbase.regionserver.HStoreFile} writing.
 *
 * @param conf
 * @param cacheConf
 * @param bloomType
 * @param maxKeys an estimate of the number of keys we expect to insert.
 *        Irrelevant if compound Bloom filters are enabled.
 * @param writer the HFile writer
 * @return the new Bloom filter, or null in case Bloom filters are disabled
 *         or when failed to create one.
 */
public static BloomFilterWriter createGeneralBloomAtWrite(Configuration conf,
    CacheConfig cacheConf, BloomType bloomType, int maxKeys,
    HFile.Writer writer) {
  if (!isGeneralBloomEnabled(conf)) {
    LOG.trace("Bloom filters are disabled by configuration for "
        + writer.getPath()
        + (conf == null ? " (configuration is null)" : ""));
    return null;
  } else if (bloomType == BloomType.NONE) {
    LOG.trace("Bloom filter is turned off for the column family");
    return null;
  }

  float err = getErrorRate(conf);

  // In case of row/column Bloom filter lookups, each lookup is an OR if two
  // separate lookups. Therefore, if each lookup's false positive rate is p,
  // the resulting false positive rate is err = 1 - (1 - p)^2, and
  // p = 1 - sqrt(1 - err).
  if (bloomType == BloomType.ROWCOL) {
    err = (float) (1 - Math.sqrt(1 - err));
  }

  int maxFold = conf.getInt(IO_STOREFILE_BLOOM_MAX_FOLD,
      MAX_ALLOWED_FOLD_FACTOR);

  // Do we support compound bloom filters?
  // In case of compound Bloom filters we ignore the maxKeys hint.
  CompoundBloomFilterWriter bloomWriter = new CompoundBloomFilterWriter(getBloomBlockSize(conf),
      err, Hash.getHashType(conf), maxFold, cacheConf.shouldCacheBloomsOnWrite(),
      bloomType == BloomType.ROWCOL ? CellComparatorImpl.COMPARATOR : null, bloomType);
  writer.addInlineBlockWriter(bloomWriter);
  return bloomWriter;
}
 
Example 17
Source Project: hbase   Source File: FixedFileTrailer.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static Class<? extends CellComparator> getComparatorClass(String comparatorClassName)
  throws IOException {
  Class<? extends CellComparator> comparatorKlass;
  // for BC
  if (comparatorClassName.equals(KeyValue.COMPARATOR.getLegacyKeyComparatorName())
    || comparatorClassName.equals(KeyValue.COMPARATOR.getClass().getName())
    || (comparatorClassName.equals("org.apache.hadoop.hbase.CellComparator"))) {
    comparatorKlass = CellComparatorImpl.class;
  } else if (comparatorClassName.equals(KeyValue.META_COMPARATOR.getLegacyKeyComparatorName())
    || comparatorClassName.equals(KeyValue.META_COMPARATOR.getClass().getName())
    || (comparatorClassName
    .equals("org.apache.hadoop.hbase.CellComparator$MetaCellComparator"))) {
    comparatorKlass = MetaCellComparator.class;
  } else if (comparatorClassName.equals("org.apache.hadoop.hbase.KeyValue$RawBytesComparator")
    || comparatorClassName.equals("org.apache.hadoop.hbase.util.Bytes$ByteArrayComparator")) {
    // When the comparator to be used is Bytes.BYTES_RAWCOMPARATOR, we just return null from here
    // Bytes.BYTES_RAWCOMPARATOR is not a CellComparator
    comparatorKlass = null;
  } else {
    // if the name wasn't one of the legacy names, maybe its a legit new kind of comparator.
    try {
      comparatorKlass = (Class<? extends CellComparator>) Class.forName(comparatorClassName);
    } catch (ClassNotFoundException e) {
      throw new IOException(e);
    }
  }
  return comparatorKlass;
}
 
Example 18
Source Project: hbase   Source File: TestDefaultMemStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Add keyvalues with a fixed memstoreTs, and checks that memstore size is decreased
 * as older keyvalues are deleted from the memstore.
 * @throws Exception
 */
@Test
public void testUpsertMemstoreSize() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  memstore = new DefaultMemStore(conf, CellComparatorImpl.COMPARATOR);
  MemStoreSize oldSize = memstore.size();

  List<Cell> l = new ArrayList<>();
  KeyValue kv1 = KeyValueTestUtil.create("r", "f", "q", 100, "v");
  KeyValue kv2 = KeyValueTestUtil.create("r", "f", "q", 101, "v");
  KeyValue kv3 = KeyValueTestUtil.create("r", "f", "q", 102, "v");

  kv1.setSequenceId(1); kv2.setSequenceId(1);kv3.setSequenceId(1);
  l.add(kv1); l.add(kv2); l.add(kv3);

  this.memstore.upsert(l, 2, null);// readpoint is 2
  MemStoreSize newSize = this.memstore.size();
  assert (newSize.getDataSize() > oldSize.getDataSize());
  //The kv1 should be removed.
  assert(memstore.getActive().getCellsCount() == 2);

  KeyValue kv4 = KeyValueTestUtil.create("r", "f", "q", 104, "v");
  kv4.setSequenceId(1);
  l.clear(); l.add(kv4);
  this.memstore.upsert(l, 3, null);
  assertEquals(newSize, this.memstore.size());
  //The kv2 should be removed.
  assert(memstore.getActive().getCellsCount() == 2);
  //this.memstore = null;
}
 
Example 19
Source Project: hbase   Source File: TestHStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Getting data from files only
 * @throws IOException
 */
@Test
public void testGet_FromFilesOnly() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null);
  //flush
  flush(3);

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(),
      qualifiers);
  //this.store.get(get, qualifiers, result);

  //Need to sort the result since multiple files
  Collections.sort(result, CellComparatorImpl.COMPARATOR);

  //Compare
  assertCheck();
}
 
Example 20
Source Project: hbase   Source File: TestHStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Getting data from memstore and files
 * @throws IOException
 */
@Test
public void testGet_FromMemStoreAndFiles() throws IOException {
  init(this.name.getMethodName());

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf2, 1, (byte[])null), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf4, 1, (byte[])null), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, (byte[])null), null);
  this.store.add(new KeyValue(row, family, qf6, 1, (byte[])null), null);

  //Get
  result = HBaseTestingUtility.getFromStoreFile(store,
      get.getRow(), qualifiers);

  //Need to sort the result since multiple files
  Collections.sort(result, CellComparatorImpl.COMPARATOR);

  //Compare
  assertCheck();
}
 
Example 21
Source Project: hbase   Source File: TestSeekOptimizations.java    License: Apache License 2.0 5 votes vote down vote up
private void prepareExpectedKVs(long latestDelTS) {
  final List<Cell> filteredKVs = new ArrayList<>();
  for (Cell kv : expectedKVs) {
    if (kv.getTimestamp() > latestDelTS || latestDelTS == -1) {
      filteredKVs.add(kv);
    }
  }
  expectedKVs = filteredKVs;
  Collections.sort(expectedKVs, CellComparatorImpl.COMPARATOR);
}
 
Example 22
Source Project: hbase   Source File: TestSeekOptimizations.java    License: Apache License 2.0 5 votes vote down vote up
public void assertKVListsEqual(String additionalMsg,
    final List<? extends Cell> expected,
    final List<? extends Cell> actual) {
  final int eLen = expected.size();
  final int aLen = actual.size();
  final int minLen = Math.min(eLen, aLen);

  int i;
  for (i = 0; i < minLen
      && PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, expected.get(i),
        actual.get(i)) == 0; ++i) {
  }

  if (additionalMsg == null) {
    additionalMsg = "";
  }
  if (!additionalMsg.isEmpty()) {
    additionalMsg = ". " + additionalMsg;
  }

  if (eLen != aLen || i != minLen) {
    throw new AssertionError(
        "Expected and actual KV arrays differ at position " + i + ": " +
        HBaseTestingUtility.safeGetAsStr(expected, i) + " (length " + eLen +") vs. " +
        HBaseTestingUtility.safeGetAsStr(actual, i) + " (length " + aLen + ")" + additionalMsg);
  }
}
 
Example 23
Source Project: hbase   Source File: TestHMobStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Getting data from memstore
 * @throws IOException
 */
@Test
public void testGetFromMemStore() throws IOException {
  final Configuration conf = HBaseConfiguration.create();
  init(name.getMethodName(), conf, false);

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, value), null);
  this.store.add(new KeyValue(row, family, qf2, 1, value), null);
  this.store.add(new KeyValue(row, family, qf3, 1, value), null);
  this.store.add(new KeyValue(row, family, qf4, 1, value), null);
  this.store.add(new KeyValue(row, family, qf5, 1, value), null);
  this.store.add(new KeyValue(row, family, qf6, 1, value), null);

  Scan scan = new Scan(get);
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
      0);

  List<Cell> results = new ArrayList<>();
  scanner.next(results);
  Collections.sort(results, CellComparatorImpl.COMPARATOR);
  scanner.close();

  //Compare
  Assert.assertEquals(expected.size(), results.size());
  for(int i=0; i<results.size(); i++) {
    // Verify the values
    Assert.assertEquals(expected.get(i), results.get(i));
  }
}
 
Example 24
Source Project: hbase   Source File: TestHMobStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Getting MOB data from files
 * @throws IOException
 */
@Test
public void testGetFromFiles() throws IOException {
  final Configuration conf = TEST_UTIL.getConfiguration();
  init(name.getMethodName(), conf, false);

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, value), null);
  this.store.add(new KeyValue(row, family, qf2, 1, value), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, value), null);
  this.store.add(new KeyValue(row, family, qf4, 1, value), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, value), null);
  this.store.add(new KeyValue(row, family, qf6, 1, value), null);
  //flush
  flush(3);

  Scan scan = new Scan(get);
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
      0);

  List<Cell> results = new ArrayList<>();
  scanner.next(results);
  Collections.sort(results, CellComparatorImpl.COMPARATOR);
  scanner.close();

  //Compare
  Assert.assertEquals(expected.size(), results.size());
  for(int i=0; i<results.size(); i++) {
    Assert.assertEquals(expected.get(i), results.get(i));
  }
}
 
Example 25
Source Project: hbase   Source File: TestHMobStore.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Getting data from memstore and files
 * @throws IOException
 */
@Test
public void testGetFromMemStoreAndFiles() throws IOException {

  final Configuration conf = HBaseConfiguration.create();

  init(name.getMethodName(), conf, false);

  //Put data in memstore
  this.store.add(new KeyValue(row, family, qf1, 1, value), null);
  this.store.add(new KeyValue(row, family, qf2, 1, value), null);
  //flush
  flush(1);

  //Add more data
  this.store.add(new KeyValue(row, family, qf3, 1, value), null);
  this.store.add(new KeyValue(row, family, qf4, 1, value), null);
  //flush
  flush(2);

  //Add more data
  this.store.add(new KeyValue(row, family, qf5, 1, value), null);
  this.store.add(new KeyValue(row, family, qf6, 1, value), null);

  Scan scan = new Scan(get);
  InternalScanner scanner = (InternalScanner) store.getScanner(scan,
      scan.getFamilyMap().get(store.getColumnFamilyDescriptor().getName()),
      0);

  List<Cell> results = new ArrayList<>();
  scanner.next(results);
  Collections.sort(results, CellComparatorImpl.COMPARATOR);
  scanner.close();

  //Compare
  Assert.assertEquals(expected.size(), results.size());
  for(int i=0; i<results.size(); i++) {
    Assert.assertEquals(expected.get(i), results.get(i));
  }
}
 
Example 26
Source Project: hbase   Source File: TestKeyValueHeap.java    License: Apache License 2.0 5 votes vote down vote up
public List<Cell> assertCells(List<Cell> expected, List<KeyValueScanner> scanners)
  throws IOException {
  // Creating KeyValueHeap
  try (KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparatorImpl.COMPARATOR)) {
    List<Cell> actual = new ArrayList<>();
    while (kvh.peek() != null) {
      actual.add(kvh.next());
    }

    assertEquals(expected, actual);
    return actual;
  }
}
 
Example 27
Source Project: hbase   Source File: TestKeyValueHeap.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testScannerLeak() throws IOException {
  // Test for unclosed scanners (HBASE-1927)

  TestScanner s4 = new TestScanner(new ArrayList<>());
  scanners.add(s4);

  // Creating KeyValueHeap
  try (KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparatorImpl.COMPARATOR)) {
    for (;;) {
      if (kvh.next() == null) {
        break;
      }
    }
    // Once the internal scanners go out of Cells, those will be removed from KVHeap's priority
    // queue and added to a Set for lazy close. The actual close will happen only on
    // KVHeap#close()
    assertEquals(4, kvh.scannersForDelayedClose.size());
    assertTrue(kvh.scannersForDelayedClose.contains(s1));
    assertTrue(kvh.scannersForDelayedClose.contains(s2));
    assertTrue(kvh.scannersForDelayedClose.contains(s3));
    assertTrue(kvh.scannersForDelayedClose.contains(s4));
  }

  for (KeyValueScanner scanner : scanners) {
    assertTrue(((TestScanner) scanner).isClosed());
  }
}
 
Example 28
Source Project: hbase   Source File: TestScanWildcardColumnTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckColumnOk() throws IOException {
  ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(
      0, VERSIONS, Long.MIN_VALUE, CellComparatorImpl.COMPARATOR);

  // Create list of qualifiers
  List<byte[]> qualifiers = new ArrayList<>(4);
  qualifiers.add(Bytes.toBytes("qualifier1"));
  qualifiers.add(Bytes.toBytes("qualifier2"));
  qualifiers.add(Bytes.toBytes("qualifier3"));
  qualifiers.add(Bytes.toBytes("qualifier4"));

  // Setting up expected result
  List<MatchCode> expected = new ArrayList<>(4);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);

  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(qualifiers.size());

  for (byte[] qualifier : qualifiers) {
    ScanQueryMatcher.MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0,
      qualifier.length, 1, KeyValue.Type.Put.getCode(), false);
    actual.add(mc);
  }

  // Compare actual with expected
  for (int i = 0; i < expected.size(); i++) {
    assertEquals(expected.get(i), actual.get(i));
  }
}
 
Example 29
Source Project: hbase   Source File: TestScanWildcardColumnTracker.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCheckColumnEnforceVersions() throws IOException {
  ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(
      0, VERSIONS, Long.MIN_VALUE, CellComparatorImpl.COMPARATOR);

  // Create list of qualifiers
  List<byte[]> qualifiers = new ArrayList<>(4);
  qualifiers.add(Bytes.toBytes("qualifier1"));
  qualifiers.add(Bytes.toBytes("qualifier1"));
  qualifiers.add(Bytes.toBytes("qualifier1"));
  qualifiers.add(Bytes.toBytes("qualifier2"));

  // Setting up expected result
  List<ScanQueryMatcher.MatchCode> expected = new ArrayList<>(4);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);
  expected.add(ScanQueryMatcher.MatchCode.SEEK_NEXT_COL);
  expected.add(ScanQueryMatcher.MatchCode.INCLUDE);

  List<MatchCode> actual = new ArrayList<>(qualifiers.size());

  long timestamp = 0;
  for (byte[] qualifier : qualifiers) {
    MatchCode mc = ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length,
      ++timestamp, KeyValue.Type.Put.getCode(), false);
    actual.add(mc);
  }

  // Compare actual with expected
  for (int i = 0; i < expected.size(); i++) {
    assertEquals(expected.get(i), actual.get(i));
  }
}
 
Example 30
Source Project: hbase   Source File: TestScannerHeartbeatMessages.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void initializeKVHeap(List<KeyValueScanner> scanners,
    List<KeyValueScanner> joinedScanners, HRegion region) throws IOException {
  this.storeHeap =
      new HeartbeatReversedKVHeap(scanners, (CellComparatorImpl) region.getCellComparator());
  if (!joinedScanners.isEmpty()) {
    this.joinedHeap = new HeartbeatReversedKVHeap(joinedScanners,
        (CellComparatorImpl) region.getCellComparator());
  }
}