org.apache.hadoop.hbase.CellComparatorImpl Java Examples

The following examples show how to use org.apache.hadoop.hbase.CellComparatorImpl. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestCompactingToCellFlatMapMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCountOfCellsAfterFlatteningByIterator() throws IOException {
  String[] keys1 = { "A", "B", "C" }; // A, B, C
  addRowsByKeysWith50Cols(memstore, keys1);
  // this should only flatten as there are no duplicates
  ((CompactingMemStore) memstore).flushInMemory();
  while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
    Threads.sleep(10);
  }
  // Just doing the cnt operation here
  MemStoreSegmentsIterator itr = new MemStoreMergerSegmentsIterator(
      ((CompactingMemStore) memstore).getImmutableSegments().getStoreSegments(),
      CellComparatorImpl.COMPARATOR, 10);
  int cnt = 0;
  try {
    while (itr.next() != null) {
      cnt++;
    }
  } finally {
    itr.close();
  }
  assertEquals("the count should be ", 150, cnt);
}
 
Example #2
Source File: TestDependentColumnFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * This shouldn't be confused with TestFilter#verifyScan
 * as expectedKeys is not the per row total, but the scan total
 *
 * @param s
 * @param expectedRows
 * @param expectedCells
 * @throws IOException
 */
private void verifyScan(Scan s, long expectedRows, long expectedCells)
throws IOException {
  InternalScanner scanner = this.region.getScanner(s);
  List<Cell> results = new ArrayList<>();
  int i = 0;
  int cells = 0;
  for (boolean done = true; done; i++) {
    done = scanner.next(results);
    Arrays.sort(results.toArray(new Cell[results.size()]),
        CellComparatorImpl.COMPARATOR);
    LOG.info("counter=" + i + ", " + results);
    if (results.isEmpty()) break;
    cells += results.size();
    assertTrue("Scanned too many rows! Only expected " + expectedRows +
        " total but already scanned " + (i+1), expectedRows > i);
    assertTrue("Expected " + expectedCells + " cells total but " +
        "already scanned " + cells, expectedCells >= cells);
    results.clear();
  }
  assertEquals("Expected " + expectedRows + " rows but scanned " + i +
      " rows", expectedRows, i);
  assertEquals("Expected " + expectedCells + " cells but scanned " + cells +
          " cells", expectedCells, cells);
}
 
Example #3
Source File: TestKeyValueHeap.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSorted() throws IOException {
  // Cases that need to be checked are:
  // 1. The "smallest" Cell is in the same scanners as current
  // 2. Current scanner gets empty

  List<Cell> expected =
    Arrays.asList(kv111, kv112, kv113, kv114, kv115, kv121, kv122, kv211, kv212, kv213);

  List<Cell> actual = assertCells(expected, scanners);

  // Check if result is sorted according to Comparator
  for (int i = 0; i < actual.size() - 1; i++) {
    int ret = CellComparatorImpl.COMPARATOR.compare(actual.get(i), actual.get(i + 1));
    assertTrue(ret < 0);
  }
}
 
Example #4
Source File: TestHFile.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testShortMidpointSameQual() {
  Cell left = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(Bytes.toBytes("a"))
    .setFamily(Bytes.toBytes("a"))
    .setQualifier(Bytes.toBytes("a"))
    .setTimestamp(11)
    .setType(Type.Maximum.getCode())
    .setValue(HConstants.EMPTY_BYTE_ARRAY)
    .build();
  Cell right = ExtendedCellBuilderFactory.create(CellBuilderType.DEEP_COPY)
    .setRow(Bytes.toBytes("a"))
    .setFamily(Bytes.toBytes("a"))
    .setQualifier(Bytes.toBytes("a"))
    .setTimestamp(9)
    .setType(Type.Maximum.getCode())
    .setValue(HConstants.EMPTY_BYTE_ARRAY)
    .build();
  Cell mid = HFileWriterImpl.getMidpoint(CellComparatorImpl.COMPARATOR, left, right);
  assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, left, mid) <= 0);
  assertTrue(PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, mid, right) == 0);
}
 
Example #5
Source File: TestKeyValueHeap.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSeek() throws IOException {
  // Cases:
  // 1. Seek Cell that is not in scanner
  // 2. Check that smallest that is returned from a seek is correct
  List<Cell> expected = Arrays.asList(kv211);

  // Creating KeyValueHeap
  try (KeyValueHeap kvh = new KeyValueHeap(scanners, CellComparatorImpl.COMPARATOR)) {
    Cell seekKv = new KeyValue(row2, fam1, null, null);
    kvh.seek(seekKv);

    List<Cell> actual = Arrays.asList(kvh.peek());

    assertEquals("Expected = " + Arrays.toString(expected.toArray()) + "\n Actual = " +
      Arrays.toString(actual.toArray()), expected, actual);
  }
}
 
Example #6
Source File: TestDataBlockEncoders.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Test whether the decompression of first key is implemented correctly.
 * @throws IOException
 */
@Test
public void testFirstKeyInBlockOnSample() throws IOException {
  List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags);

  for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
    if (encoding.getEncoder() == null) {
      continue;
    }
    DataBlockEncoder encoder = encoding.getEncoder();
    ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv,
        getEncodingContext(Compression.Algorithm.NONE, encoding), this.useOffheapData);
    Cell key = encoder.getFirstKeyCellInBlock(new SingleByteBuff(encodedBuffer));
    KeyValue firstKv = sampleKv.get(0);
    if (0 != PrivateCellUtil.compareKeyIgnoresMvcc(CellComparatorImpl.COMPARATOR, key, firstKv)) {
      int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(key, firstKv, false, true);
      fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix));
    }
  }
}
 
Example #7
Source File: TestProtobufUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testToCell() {
  KeyValue kv1 =
      new KeyValue(Bytes.toBytes("aaa"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv2 =
      new KeyValue(Bytes.toBytes("bbb"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  KeyValue kv3 =
      new KeyValue(Bytes.toBytes("ccc"), Bytes.toBytes("f1"), Bytes.toBytes("q1"), new byte[30]);
  byte[] arr = new byte[kv1.getLength() + kv2.getLength() + kv3.getLength()];
  System.arraycopy(kv1.getBuffer(), kv1.getOffset(), arr, 0, kv1.getLength());
  System.arraycopy(kv2.getBuffer(), kv2.getOffset(), arr, kv1.getLength(), kv2.getLength());
  System.arraycopy(kv3.getBuffer(), kv3.getOffset(), arr, kv1.getLength() + kv2.getLength(),
    kv3.getLength());
  ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length);
  dbb.put(arr);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, kv1.getLength(), kv2.getLength());
  CellProtos.Cell cell = ProtobufUtil.toCell(offheapKV);
  Cell newOffheapKV =
      ProtobufUtil.toCell(ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY), cell);
  assertTrue(CellComparatorImpl.COMPARATOR.compare(offheapKV, newOffheapKV) == 0);
}
 
Example #8
Source File: HFileContext.java    From hbase with Apache License 2.0 6 votes vote down vote up
HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
             Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
             int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
             Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
             byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
  this.usesHBaseChecksum = useHBaseChecksum;
  this.includesMvcc =  includesMvcc;
  this.includesTags = includesTags;
  this.compressAlgo = compressAlgo;
  this.compressTags = compressTags;
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
  this.blocksize = blockSize;
  if (encoding != null) {
    this.encoding = encoding;
  }
  this.cryptoContext = cryptoContext;
  this.fileCreateTime = fileCreateTime;
  this.hfileName = hfileName;
  this.columnFamily = columnFamily;
  this.tableName = tableName;
  // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
  // be the meta table comparator. Comparators are per table.
  this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
    CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
 
Example #9
Source File: TestDefaultStoreEngine.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCustomParts() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName());
  conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
      DummyCompactionPolicy.class.getName());
  conf.set(DefaultStoreEngine.DEFAULT_STORE_FLUSHER_CLASS_KEY,
      DummyStoreFlusher.class.getName());
  HStore mockStore = Mockito.mock(HStore.class);
  Mockito.when(mockStore.getRegionInfo()).thenReturn(RegionInfoBuilder.FIRST_META_REGIONINFO);
  StoreEngine<?, ?, ?, ?> se = StoreEngine.create(mockStore, conf, CellComparatorImpl.COMPARATOR);
  Assert.assertTrue(se instanceof DefaultStoreEngine);
  Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy);
  Assert.assertTrue(se.getStoreFlusher() instanceof DummyStoreFlusher);
  Assert.assertTrue(se.getCompactor() instanceof DummyCompactor);
}
 
Example #10
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleVersionsSimple() throws Exception {
  DefaultMemStore m = new DefaultMemStore(new Configuration(), CellComparatorImpl.COMPARATOR);
  byte [] row = Bytes.toBytes("testRow");
  byte [] family = Bytes.toBytes("testFamily");
  byte [] qf = Bytes.toBytes("testQualifier");
  long [] stamps = {1,2,3};
  byte [][] values = {Bytes.toBytes("value0"), Bytes.toBytes("value1"),
      Bytes.toBytes("value2")};
  KeyValue key0 = new KeyValue(row, family, qf, stamps[0], values[0]);
  KeyValue key1 = new KeyValue(row, family, qf, stamps[1], values[1]);
  KeyValue key2 = new KeyValue(row, family, qf, stamps[2], values[2]);

  m.add(key0, null);
  m.add(key1, null);
  m.add(key2, null);

  assertTrue("Expected memstore to hold 3 values, actually has " +
      m.getActive().getCellsCount(), m.getActive().getCellsCount() == 3);
}
 
Example #11
Source File: TestReplicationWALEntryFilters.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void assertEquals(Entry e1, Entry e2) {
  Assert.assertEquals(e1 == null, e2 == null);
  if (e1 == null) {
    return;
  }

  // do not compare WALKeys

  // compare kvs
  Assert.assertEquals(e1.getEdit() == null, e2.getEdit() == null);
  if (e1.getEdit() == null) {
    return;
  }
  List<Cell> cells1 = e1.getEdit().getCells();
  List<Cell> cells2 = e2.getEdit().getCells();
  Assert.assertEquals(cells1.size(), cells2.size());
  for (int i = 0; i < cells1.size(); i++) {
    CellComparatorImpl.COMPARATOR.compare(cells1.get(i), cells2.get(i));
  }
}
 
Example #12
Source File: FixedFileTrailer.java    From hbase with Apache License 2.0 6 votes vote down vote up
static CellComparator createComparator(String comparatorClassName) throws IOException {
  if (comparatorClassName.equals(CellComparatorImpl.COMPARATOR.getClass().getName())) {
    return CellComparatorImpl.COMPARATOR;
  } else if (comparatorClassName.equals(
    CellComparatorImpl.META_COMPARATOR.getClass().getName())) {
    return CellComparatorImpl.META_COMPARATOR;
  }
  try {
    Class<? extends CellComparator> comparatorClass = getComparatorClass(comparatorClassName);
    if (comparatorClass != null) {
      return comparatorClass.getDeclaredConstructor().newInstance();
    }
    LOG.warn("No Comparator class for " + comparatorClassName + ". Returning Null.");
    return null;
  } catch (Exception e) {
    throw new IOException("Comparator class " + comparatorClassName + " is not instantiable", e);
  }
}
 
Example #13
Source File: TestScanWildcardColumnTracker.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void DisabledTestCheckColumnWrongOrder() {
  ScanWildcardColumnTracker tracker = new ScanWildcardColumnTracker(
      0, VERSIONS, Long.MIN_VALUE, CellComparatorImpl.COMPARATOR);

  // Create list of qualifiers
  List<byte[]> qualifiers = new ArrayList<>(2);
  qualifiers.add(Bytes.toBytes("qualifier2"));
  qualifiers.add(Bytes.toBytes("qualifier1"));

  try {
    for (byte[] qualifier : qualifiers) {
      ScanQueryMatcher.checkColumn(tracker, qualifier, 0, qualifier.length, 1,
        KeyValue.Type.Put.getCode(), false);
    }
    fail();
  } catch (IOException e) {
    // expected
  }
}
 
Example #14
Source File: TestStripeCompactionPolicy.java    From hbase with Apache License 2.0 6 votes vote down vote up
/** Verify arbitrary flush. */
protected void verifyFlush(StripeCompactionPolicy policy, StripeInformationProvider si,
    KeyValue[] input, KeyValue[][] expected, byte[][] boundaries) throws IOException {
  StoreFileWritersCapture writers = new StoreFileWritersCapture();
  StripeStoreFlusher.StripeFlushRequest req =
    policy.selectFlush(CellComparatorImpl.COMPARATOR, si, input.length);
  StripeMultiFileWriter mw = req.createWriter();
  mw.init(null, writers);
  for (KeyValue kv : input) {
    mw.append(kv);
  }
  boolean hasMetadata = boundaries != null;
  mw.commitWriters(0, false);
  writers.verifyKvs(expected, true, hasMetadata);
  if (hasMetadata) {
    writers.verifyBoundaries(boundaries);
  }
}
 
Example #15
Source File: OrderByTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortOrderForLeadingDescVarLengthColWithNonNullFollowing() throws SQLException {
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement().execute("CREATE TABLE t (k1 VARCHAR, k2 VARCHAR NOT NULL, CONSTRAINT pk PRIMARY KEY (k1 DESC,k2))");
    conn.createStatement().execute("UPSERT INTO t VALUES ('a','x')");
    conn.createStatement().execute("UPSERT INTO t VALUES ('ab', 'x')");

    Iterator<Pair<byte[],List<Cell>>> dataIterator = PhoenixRuntime.getUncommittedDataIterator(conn);
    List<Cell> kvs = dataIterator.next().getSecond();
    Collections.sort(kvs, CellComparatorImpl.COMPARATOR);
    Cell first = kvs.get(0);
    assertEquals("ab", Bytes.toString(SortOrder.invert(first.getRowArray(), first.getRowOffset(), 2)));
    Cell second = kvs.get(1);
    assertEquals("a", Bytes.toString(SortOrder.invert(second.getRowArray(), second.getRowOffset(), 1)));
}
 
Example #16
Source File: OrderByTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortOrderForSingleDescVarLengthCol() throws SQLException {
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement().execute("CREATE TABLE t (k VARCHAR PRIMARY KEY DESC)");
    conn.createStatement().execute("UPSERT INTO t VALUES ('a')");
    conn.createStatement().execute("UPSERT INTO t VALUES ('ab')");

    Iterator<Pair<byte[],List<Cell>>> dataIterator = PhoenixRuntime.getUncommittedDataIterator(conn);
    List<Cell> kvs = dataIterator.next().getSecond();
    Collections.sort(kvs, CellComparatorImpl.COMPARATOR);
    Cell first = kvs.get(0);
    assertEquals("ab", Bytes.toString(SortOrder.invert(first.getRowArray(), first.getRowOffset(), first.getRowLength()-1)));
    Cell second = kvs.get(1);
    assertEquals("a", Bytes.toString(SortOrder.invert(second.getRowArray(), second.getRowOffset(), second.getRowLength()-1)));
}
 
Example #17
Source File: OrderByTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testSortOrderForLeadingDescVarLengthColWithNullFollowing() throws SQLException {
    Connection conn = DriverManager.getConnection(getUrl());
    conn.createStatement().execute("CREATE TABLE t (k1 VARCHAR, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1 DESC,k2))");
    conn.createStatement().execute("UPSERT INTO t VALUES ('a')");
    conn.createStatement().execute("UPSERT INTO t VALUES ('ab')");

    Iterator<Pair<byte[],List<Cell>>> dataIterator = PhoenixRuntime.getUncommittedDataIterator(conn);
    List<Cell> kvs = dataIterator.next().getSecond();
    Collections.sort(kvs, CellComparatorImpl.COMPARATOR);
    Cell first = kvs.get(0);
    assertEquals("ab", Bytes.toString(SortOrder.invert(first.getRowArray(), first.getRowOffset(), first.getRowLength()-1)));
    Cell second = kvs.get(1);
    assertEquals("a", Bytes.toString(SortOrder.invert(second.getRowArray(), second.getRowOffset(), second.getRowLength()-1)));
}
 
Example #18
Source File: TestFixedFileTrailer.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void checkLoadedTrailer(int version, FixedFileTrailer expected,
    FixedFileTrailer loaded) throws IOException {
  assertEquals(version, loaded.getMajorVersion());
  assertEquals(expected.getDataIndexCount(), loaded.getDataIndexCount());

  assertEquals(Math.min(expected.getEntryCount(),
      version == 1 ? Integer.MAX_VALUE : Long.MAX_VALUE),
      loaded.getEntryCount());

  if (version == 1) {
    assertEquals(expected.getFileInfoOffset(), loaded.getFileInfoOffset());
  }

  if (version == 2) {
    assertEquals(expected.getLastDataBlockOffset(),
        loaded.getLastDataBlockOffset());
    assertEquals(expected.getNumDataIndexLevels(),
        loaded.getNumDataIndexLevels());
    assertEquals(expected.createComparator().getClass().getName(),
        loaded.createComparator().getClass().getName());
    assertEquals(expected.getFirstDataBlockOffset(),
        loaded.getFirstDataBlockOffset());
    assertTrue(
        expected.createComparator() instanceof CellComparatorImpl);
    assertEquals(expected.getUncompressedDataIndexSize(),
        loaded.getUncompressedDataIndexSize());
  }

  assertEquals(expected.getLoadOnOpenDataOffset(),
      loaded.getLoadOnOpenDataOffset());
  assertEquals(expected.getMetaIndexCount(), loaded.getMetaIndexCount());

  assertEquals(expected.getTotalUncompressedBytes(),
      loaded.getTotalUncompressedBytes());
}
 
Example #19
Source File: IndexMemStore.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public IndexMemStore() {
  this(new DelegateComparator(new CellComparatorImpl()){
      @Override
      public int compare(Cell leftCell, Cell rightCell) {
          return super.compare(leftCell, rightCell, true);
      }
  });
}
 
Example #20
Source File: TestHFileWriterV3WithDataEncoders.java    From hbase with Apache License 2.0 5 votes vote down vote up
private long scanBlocks(int entryCount, HFileContext context, List<KeyValue> keyValues,
    FSDataInputStream fsdis, FixedFileTrailer trailer, HFileContext meta,
    HFileBlock.FSReader blockReader, int entriesRead, int blocksRead,
    DataBlockEncoder encoder) throws IOException {
  // Scan blocks the way the reader would scan them
  fsdis.seek(0);
  long curBlockPos = 0;
  while (curBlockPos <= trailer.getLastDataBlockOffset()) {
    HFileBlockDecodingContext ctx = blockReader.getBlockDecodingContext();
    HFileBlock block = blockReader.readBlockData(curBlockPos, -1, false, false, true)
      .unpack(context, blockReader);
    Assert.assertEquals(BlockType.ENCODED_DATA, block.getBlockType());
    ByteBuff origBlock = block.getBufferReadOnly();
    int pos = block.headerSize() + DataBlockEncoding.ID_SIZE;
    origBlock.position(pos);
    origBlock.limit(pos + block.getUncompressedSizeWithoutHeader() - DataBlockEncoding.ID_SIZE);
    ByteBuff buf =  origBlock.slice();
    DataBlockEncoder.EncodedSeeker seeker =
      encoder.createSeeker(encoder.newDataBlockDecodingContext(meta));
    seeker.setCurrentBuffer(buf);
    Cell res = seeker.getCell();
    KeyValue kv = keyValues.get(entriesRead);
    Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(res, kv));
    ++entriesRead;
    while(seeker.next()) {
      res = seeker.getCell();
      kv = keyValues.get(entriesRead);
      Assert.assertEquals(0, CellComparatorImpl.COMPARATOR.compare(res, kv));
      ++entriesRead;
    }
    ++blocksRead;
    curBlockPos += block.getOnDiskSizeWithHeader();
  }
  LOG.info("Finished reading: entries={}, blocksRead = {}", entriesRead, blocksRead);
  Assert.assertEquals(entryCount, entriesRead);
  return curBlockPos;
}
 
Example #21
Source File: TestFixedFileTrailer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testTrailerForV2NonPBCompatibility() throws Exception {
  if (version == 2) {
    FixedFileTrailer t = new FixedFileTrailer(version,
        HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
    t.setDataIndexCount(3);
    t.setEntryCount(((long) Integer.MAX_VALUE) + 1);
    t.setLastDataBlockOffset(291);
    t.setNumDataIndexLevels(3);
    t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
    t.setFirstDataBlockOffset(9081723123L); // Completely unrealistic.
    t.setUncompressedDataIndexSize(827398717L); // Something random.
    t.setLoadOnOpenOffset(128);
    t.setMetaIndexCount(7);
    t.setTotalUncompressedBytes(129731987);

    {
      DataOutputStream dos = new DataOutputStream(baos); // Limited scope.
      serializeAsWritable(dos, t);
      dos.flush();
      assertEquals(FixedFileTrailer.getTrailerSize(version), dos.size());
    }

    byte[] bytes = baos.toByteArray();
    baos.reset();
    assertEquals(bytes.length, FixedFileTrailer.getTrailerSize(version));

    ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
    {
      DataInputStream dis = new DataInputStream(bais);
      FixedFileTrailer t2 = new FixedFileTrailer(version,
          HFileReaderImpl.MINOR_VERSION_NO_CHECKSUM);
      t2.deserialize(dis);
      assertEquals(-1, bais.read()); // Ensure we have read everything.
      checkLoadedTrailer(version, t, t2);
    }
  }
}
 
Example #22
Source File: TestRowIndexV1DataEncoder.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void writeDataToHFile(Path hfilePath, int entryCount) throws IOException {
  HFileContext context =
    new HFileContextBuilder().withBlockSize(1024).withDataBlockEncoding(dataBlockEncoding)
      .withCellComparator(CellComparatorImpl.COMPARATOR).build();
  CacheConfig cacheConfig = new CacheConfig(conf);
  HFile.Writer writer =
    new HFile.WriterFactory(conf, cacheConfig).withPath(fs, hfilePath).withFileContext(context)
      .create();

  List<KeyValue> keyValues = new ArrayList<>(entryCount);

  writeKeyValues(entryCount, writer, keyValues);

  FSDataInputStream fsdis = fs.open(hfilePath);

  long fileSize = fs.getFileStatus(hfilePath).getLen();
  FixedFileTrailer trailer = FixedFileTrailer.readFromStream(fsdis, fileSize);

  // HBASE-23788
  // kv size = 24 bytes, block size = 1024 bytes
  // per row encoded data written = (4 (Row index) + 24 (Cell size) + 1 (MVCC)) bytes = 29 bytes
  // creating block size of (29 * 36) bytes = 1044 bytes
  // Number of blocks = ceil((29 * 10000) / 1044) = 278
  // Without the patch it would have produced 244 blocks (each block of 1236 bytes)
  // Earlier this would create blocks ~20% greater than the block size of 1024 bytes
  // After this patch actual block size is ~2% greater than the block size of 1024 bytes
  Assert.assertEquals(278, trailer.getDataIndexCount());
}
 
Example #23
Source File: OrderByTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
  public void testSortOrderForSingleDescTimestampCol() throws SQLException {
      Connection conn = DriverManager.getConnection(getUrl());
      conn.createStatement().execute("CREATE TABLE t (k TIMESTAMP PRIMARY KEY DESC)");
      conn.createStatement().execute("UPSERT INTO t VALUES ('2016-01-04 13:11:51.631')");

      Iterator<Pair<byte[], List<Cell>>> dataIterator = PhoenixRuntime
          .getUncommittedDataIterator(conn);
      List<Cell> kvs = dataIterator.next().getSecond();
      Collections.sort(kvs, CellComparatorImpl.COMPARATOR);
      Cell first = kvs.get(0);
      long millisDeserialized = PDate.INSTANCE.getCodec().decodeLong(first.getRowArray(),
          first.getRowOffset(), SortOrder.DESC);
      assertEquals(1451913111631L, millisDeserialized);
}
 
Example #24
Source File: TestIndexMemStore.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Test
public void testCorrectOverwritting() throws Exception {
  IndexMemStore store = new IndexMemStore(new DelegateComparator(new CellComparatorImpl()){
      @Override
      public int compare(Cell leftCell, Cell rightCell) {
          return super.compare(leftCell, rightCell, true);
      }
  });
  long ts = 10;
  KeyValue kv = new KeyValue(row, family, qual, ts, Type.Put, val);
  kv.setSequenceId(2);
  KeyValue kv2 = new KeyValue(row, family, qual, ts, Type.Put, val2);
  kv2.setSequenceId(0);
  store.add(kv, true);
  // adding the exact same kv shouldn't change anything stored if not overwritting
  store.add(kv2, false);
  ReseekableScanner scanner = store.getScanner();
  KeyValue first = KeyValueUtil.createFirstOnRow(row);
  scanner.seek(first);
  assertTrue("Overwrote kv when specifically not!", kv == scanner.next());
  scanner.close();

  // now when we overwrite, we should get the newer one
  store.add(kv2, true);
  scanner = store.getScanner();
  scanner.seek(first);
  assertTrue("Didn't overwrite kv when specifically requested!", kv2 == scanner.next());
  scanner.close();
}
 
Example #25
Source File: LocalIndexStoreFileScanner.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public LocalIndexStoreFileScanner(IndexHalfStoreFileReader reader, boolean cacheBlocks, boolean pread,
        boolean isCompaction, long readPt, long scannerOrder,
        boolean canOptimizeForNonNullColumn) {
    super(reader, reader.getScanner(cacheBlocks, pread, isCompaction), !isCompaction, reader
            .getHFileReader().hasMVCCInfo(), readPt, scannerOrder, canOptimizeForNonNullColumn);
    this.reader = reader;
    this.changeBottomKeys =
            this.reader.getRegionInfo().getStartKey().length == 0
                    && this.reader.getSplitRow().length != this.reader.getOffset();
    this.comparator = (CellComparatorImpl)getComparator();
}
 
Example #26
Source File: TestFixedFileTrailer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testComparatorIsHBase1Compatible() {
  FixedFileTrailer t = new FixedFileTrailer(version, HFileReaderImpl.PBUF_TRAILER_MINOR_VERSION);
  t.setComparatorClass(CellComparatorImpl.COMPARATOR.getClass());
  assertEquals(CellComparatorImpl.COMPARATOR.getClass().getName(), t.getComparatorClassName());
  HFileProtos.FileTrailerProto pb = t.toProtobuf();
  assertEquals(KeyValue.COMPARATOR.getClass().getName(), pb.getComparatorClassName());
  t.setComparatorClass(CellComparatorImpl.MetaCellComparator.META_COMPARATOR.getClass());
  pb = t.toProtobuf();
  assertEquals(KeyValue.META_COMPARATOR.getClass().getName(),
      pb.getComparatorClassName());
}
 
Example #27
Source File: TestCompoundBloomFilter.java    From hbase with Apache License 2.0 5 votes vote down vote up
private List<KeyValue> createSortedKeyValues(Random rand, int n) {
  List<KeyValue> kvList = new ArrayList<>(n);
  for (int i = 0; i < n; ++i)
    kvList.add(RandomKeyValueUtil.randomKeyValue(rand));
  Collections.sort(kvList, CellComparatorImpl.COMPARATOR);
  return kvList;
}
 
Example #28
Source File: TestCompactingToCellFlatMapMemStore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override public void setUp() throws Exception {

    compactingSetUp();
    this.conf = HBaseConfiguration.create();

    // set memstore to do data compaction
    conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY,
        String.valueOf(MemoryCompactionPolicy.EAGER));
    conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.02);
    this.memstore =
        new MyCompactingMemStore(conf, CellComparatorImpl.COMPARATOR, store,
            regionServicesForStores, MemoryCompactionPolicy.EAGER);
  }
 
Example #29
Source File: TestStripeStoreFileManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static StripeStoreFileManager createManager(
    ArrayList<HStoreFile> sfs, Configuration conf) throws Exception {
  StripeStoreConfig config = new StripeStoreConfig(
      conf, Mockito.mock(StoreConfigInformation.class));
  StripeStoreFileManager result = new StripeStoreFileManager(CellComparatorImpl.COMPARATOR, conf,
      config);
  result.loadFiles(sfs);
  return result;
}
 
Example #30
Source File: TestScannerHeartbeatMessages.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void initializeKVHeap(List<KeyValueScanner> scanners,
    List<KeyValueScanner> joinedScanners, HRegion region) throws IOException {
  this.storeHeap =
      new HeartbeatReversedKVHeap(scanners, (CellComparatorImpl) region.getCellComparator());
  if (!joinedScanners.isEmpty()) {
    this.joinedHeap = new HeartbeatReversedKVHeap(joinedScanners,
        (CellComparatorImpl) region.getCellComparator());
  }
}