Java Code Examples for org.apache.hadoop.hbase.CellComparator#getInstance()

HFileContext(boolean useHBaseChecksum, boolean includesMvcc, boolean includesTags,
             Compression.Algorithm compressAlgo, boolean compressTags, ChecksumType checksumType,
             int bytesPerChecksum, int blockSize, DataBlockEncoding encoding,
             Encryption.Context cryptoContext, long fileCreateTime, String hfileName,
             byte[] columnFamily, byte[] tableName, CellComparator cellComparator) {
  this.usesHBaseChecksum = useHBaseChecksum;
  this.includesMvcc =  includesMvcc;
  this.includesTags = includesTags;
  this.compressAlgo = compressAlgo;
  this.compressTags = compressTags;
  this.checksumType = checksumType;
  this.bytesPerChecksum = bytesPerChecksum;
  this.blocksize = blockSize;
  if (encoding != null) {
    this.encoding = encoding;
  }
  this.cryptoContext = cryptoContext;
  this.fileCreateTime = fileCreateTime;
  this.hfileName = hfileName;
  this.columnFamily = columnFamily;
  this.tableName = tableName;
  // If no cellComparator specified, make a guess based off tablename. If hbase:meta, then should
  // be the meta table comparator. Comparators are per table.
  this.cellComparator = cellComparator != null ? cellComparator : this.tableName != null ?
    CellComparatorImpl.getCellComparator(this.tableName) : CellComparator.getInstance();
}
 

@Test
public void testReadVersionWithRawAndFilter() throws IOException {
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, Long.MAX_VALUE,
          KeepDeletedCells.FALSE, HConstants.DEFAULT_BLOCKSIZE, 0
          , CellComparator.getInstance(), false);
  KeyValue [] kvs = new KeyValue[] {
          create("R1", "cf", "a", 3, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 2, KeyValue.Type.Put, "dont-care"),
          create("R1", "cf", "a", 1, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = Arrays.asList(
    new KeyValueScanner[]{
      new KeyValueScanFixture(CellComparator.getInstance(), kvs)
    });

  BinaryComparator comp = new BinaryComparator(Bytes.toBytes("a"));
  Filter filter = new QualifierFilter(CompareOperator.EQUAL, comp);
  Scan scanSpec = new Scan().withStartRow(Bytes.toBytes("R1")).readVersions(2).setRaw(true);
  scanSpec.setFilter(filter);
  try (StoreScanner scan = new StoreScanner(scanSpec, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scan.next(results));
    assertEquals(2, results.size());
  }
}
 

@Before
public void setUp() throws Exception {
  this.conf = HBaseConfiguration.create();
  row1 = Bytes.toBytes("row1");
  row2 = Bytes.toBytes("row2");
  row3 = Bytes.toBytes("row3");
  fam1 = Bytes.toBytes("fam1");
  fam2 = Bytes.toBytes("fam2");
  col1 = Bytes.toBytes("col1");
  col2 = Bytes.toBytes("col2");
  col3 = Bytes.toBytes("col3");
  col4 = Bytes.toBytes("col4");
  col5 = Bytes.toBytes("col5");

  data = Bytes.toBytes("data");

  // Create Get
  get = new Get(row1);
  get.addFamily(fam1);
  get.addColumn(fam2, col2);
  get.addColumn(fam2, col4);
  get.addColumn(fam2, col5);
  this.scan = new Scan(get);

  rowComparator = CellComparator.getInstance();
}
 

/**
 * Ensure that expired delete family markers don't override valid puts
 */
@Test
public void testExpiredDeleteFamily() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now-1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now-10, KeyValue.Type.Put,
      "dont-care"),
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  // scanner with ttl equal to 500
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(1, results.size());
    assertEquals(kvs[1], results.get(0));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 

@Before
public void setup() throws IOException {
  Configuration conf = new Configuration();
  HBaseTestingUtility hbaseUtility = new HBaseTestingUtility(conf);
  TableDescriptorBuilder tableDescriptorBuilder =
    TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE));
  ColumnFamilyDescriptor columnFamilyDescriptor =
    ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(FAMILY)).build();
  tableDescriptorBuilder.setColumnFamily(columnFamilyDescriptor);

  RegionInfo info = RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE)).build();
  Path rootPath = hbaseUtility.getDataTestDir(ROOT_SUB_PATH);
  this.wal = HBaseTestingUtility.createWal(conf, rootPath, info);
  this.region = HRegion.createHRegion(info, rootPath, conf,
    tableDescriptorBuilder.build(), this.wal, true);
  this.store = new HStore(this.region, columnFamilyDescriptor, conf, false);
  this.comparator = CellComparator.getInstance();
  this.compactionKVMax = HConstants.COMPACTION_KV_MAX_DEFAULT;
}
 

@Override
@Before
public void setUp() throws Exception {
  compactingSetUp();
  this.memstore = new MyCompactingMemStore(HBaseConfiguration.create(), CellComparator.getInstance(),
      store, regionServicesForStores, MemoryCompactionPolicy.EAGER);
  ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP);
}
 

@Test
public void testPreadNotEnabledForCompactionStoreScanners() throws Exception {
  long now = System.currentTimeMillis();
  KeyValue[] kvs = new KeyValue[] {
    new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null, now - 1000,
      KeyValue.Type.DeleteFamily),
    create("R1", "cf", "a", now - 10, KeyValue.Type.Put, "dont-care"), };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner storeScanner = new StoreScanner(scanInfo, -1,
    ScanType.COMPACT_RETAIN_DELETES, scanners)) {
    assertFalse(storeScanner.isScanUsePread());
  }
}
 

@Test
public void testWildCardTtlScan() throws IOException {
  long now = System.currentTimeMillis();
  KeyValue [] kvs = new KeyValue[] {
      create("R1", "cf", "a", now-1000, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R1", "cf", "d", now-10000, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "a", now, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "b", now-10, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-200, KeyValue.Type.Put, "dont-care"),
      create("R2", "cf", "c", now-1000, KeyValue.Type.Put, "dont-care")
  };
  List<KeyValueScanner> scanners = scanFixture(kvs);
  Scan scan = new Scan();
  scan.readVersions(1);
  ScanInfo scanInfo = new ScanInfo(CONF, CF, 0, 1, 500, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, 0, CellComparator.getInstance(), false);
  try (StoreScanner scanner = new StoreScanner(scan, scanInfo, null, scanners)) {
    List<Cell> results = new ArrayList<>();
    assertEquals(true, scanner.next(results));
    assertEquals(2, results.size());
    assertEquals(kvs[1], results.get(0));
    assertEquals(kvs[2], results.get(1));
    results.clear();

    assertEquals(true, scanner.next(results));
    assertEquals(3, results.size());
    assertEquals(kvs[4], results.get(0));
    assertEquals(kvs[5], results.get(1));
    assertEquals(kvs[6], results.get(2));
    results.clear();

    assertEquals(false, scanner.next(results));
  }
}
 

/**
 * A simple test which verifies the 3 possible states when scanning across snapshot.
 *
 * @throws IOException
 * @throws CloneNotSupportedException
 */
@Override
@Test
public void testScanAcrossSnapshot2() throws IOException, CloneNotSupportedException {
  // we are going to the scanning across snapshot with two kvs
  // kv1 should always be returned before kv2
  final byte[] one = Bytes.toBytes(1);
  final byte[] two = Bytes.toBytes(2);
  final byte[] f = Bytes.toBytes("f");
  final byte[] q = Bytes.toBytes("q");
  final byte[] v = Bytes.toBytes(3);

  final KeyValue kv1 = new KeyValue(one, f, q, 10, v);
  final KeyValue kv2 = new KeyValue(two, f, q, 10, v);

  // use case 1: both kvs in kvset
  this.memstore.add(kv1.clone(), null);
  this.memstore.add(kv2.clone(), null);
  verifyScanAcrossSnapshot2(kv1, kv2);

  // use case 2: both kvs in snapshot
  this.memstore.snapshot();
  verifyScanAcrossSnapshot2(kv1, kv2);

  // use case 3: first in snapshot second in kvset
  this.memstore = new CompactingMemStore(HBaseConfiguration.create(),
      CellComparator.getInstance(), store, regionServicesForStores,
      MemoryCompactionPolicy.EAGER);
  this.memstore.add(kv1.clone(), null);
  // As compaction is starting in the background the repetition
  // of the k1 might be removed BUT the scanners created earlier
  // should look on the OLD MutableCellSetSegment, so this should be OK...
  this.memstore.snapshot();
  this.memstore.add(kv2.clone(), null);
  verifyScanAcrossSnapshot2(kv1,kv2);
}
 

@Before
public void setUp() throws Exception {
  // create array of Cells to bass to the CellFlatMap under CellSet
  final byte[] one = Bytes.toBytes(15);
  final byte[] two = Bytes.toBytes(25);
  final byte[] three = Bytes.toBytes(35);
  final byte[] four = Bytes.toBytes(45);

  final byte[] f = Bytes.toBytes("f");
  final byte[] q = Bytes.toBytes("q");
  final byte[] v = Bytes.toBytes(4);

  final KeyValue kv1 = new KeyValue(one, f, q, 10, v);
  final KeyValue kv2 = new KeyValue(two, f, q, 20, v);
  final KeyValue kv3 = new KeyValue(three, f, q, 30, v);
  final KeyValue kv4 = new KeyValue(four, f, q, 40, v);
  lowerOuterCell = new KeyValue(Bytes.toBytes(10), f, q, 10, v);
  upperOuterCell = new KeyValue(Bytes.toBytes(50), f, q, 10, v);
  ascCells = new Cell[] {kv1,kv2,kv3,kv4};
  ascCbOnHeap = new CellArrayMap(CellComparator.getInstance(), ascCells,0, NUM_OF_CELLS,false);
  descCells = new Cell[] {kv4,kv3,kv2,kv1};
  descCbOnHeap = new CellArrayMap(CellComparator.getInstance(), descCells,0, NUM_OF_CELLS,true);

  CONF.setBoolean(MemStoreLAB.USEMSLAB_KEY, true);
  CONF.setFloat(MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY, 0.2f);
  ChunkCreator.chunkPoolDisabled = false;

  // create ascending and descending CellChunkMaps
  // according to parameter, once built with normal chunks and at second with small chunks
  ascCCM = setUpCellChunkMap(true);
  descCCM = setUpCellChunkMap(false);

  if (smallChunks) {    // check jumbo chunks as well
    ascCCM = setUpJumboCellChunkMap(true);
  }
}
 

public void testKeyValueScanFixture() throws IOException {
  KeyValue kvs[] = new KeyValue[]{
      KeyValueTestUtil.create("RowA", "family", "qf1",
          1, KeyValue.Type.Put, "value-1"),
      KeyValueTestUtil.create("RowA", "family", "qf2",
          1, KeyValue.Type.Put, "value-2"),
      KeyValueTestUtil.create("RowB", "family", "qf1",
          10, KeyValue.Type.Put, "value-10")
  };
  KeyValueScanner scan = new KeyValueScanFixture(CellComparator.getInstance(), kvs);

  KeyValue kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  // should seek to this:
  assertTrue(scan.seek(kv));
  Cell res = scan.peek();
  assertEquals(kvs[0], res);

  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowB"));
  assertTrue(scan.seek(kv));
  res = scan.peek();
  assertEquals(kvs[2], res);

  // ensure we pull things out properly:
  kv = KeyValueUtil.createFirstOnRow(Bytes.toBytes("RowA"));
  assertTrue(scan.seek(kv));
  assertEquals(kvs[0], scan.peek());
  assertEquals(kvs[0], scan.next());
  assertEquals(kvs[1], scan.peek());
  assertEquals(kvs[1], scan.next());
  assertEquals(kvs[2], scan.peek());
  assertEquals(kvs[2], scan.next());
  assertEquals(null, scan.peek());
  assertEquals(null, scan.next());
}
 

private ScanInfo getScanInfo() {
  int oldMaxVersions = 1;
  int oldMinVersions = 0;
  long oldTTL = 10000;

  return new ScanInfo(conf, Bytes.toBytes("cf"), oldMinVersions, oldMaxVersions, oldTTL,
  KeepDeletedCells.FALSE, HConstants.FOREVER, 1000,
    CellComparator.getInstance(), true);
}
 

public CollectionBackedScanner(SortedSet<Cell> set) {
  this(set, CellComparator.getInstance());
}
 

private CellChunkMap setUpJumboCellChunkMap(boolean asc) {
  int smallChunkSize = SMALL_CHUNK_SIZE+8;
  // allocate new chunks and use the data JUMBO chunk to hold the full data of the cells
  // and the normal index chunk to hold the cell-representations
  Chunk dataJumboChunk =
      chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP, smallChunkSize);
  Chunk idxChunk  = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
  // the array of index chunks to be used as a basis for CellChunkMap
  Chunk[] chunkArray = new Chunk[8];  // according to test currently written 8 is way enough
  int chunkArrayIdx = 0;
  chunkArray[chunkArrayIdx++] = idxChunk;

  ByteBuffer idxBuffer = idxChunk.getData();  // the buffers of the chunks
  ByteBuffer dataBuffer = dataJumboChunk.getData();
  int dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;          // offset inside data buffer
  int idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;           // skip the space for chunk ID

  Cell[] cellArray = asc ? ascCells : descCells;

  for (Cell kv: cellArray) {
    int dataStartOfset = dataOffset;
    dataOffset = KeyValueUtil.appendTo(kv, dataBuffer, dataOffset, false); // write deep cell data

    // do we have enough space to write the cell-representation on the index chunk?
    if (idxOffset + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkCreator.getChunkSize()) {
      // allocate more index chunks if needed
      idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
      idxBuffer = idxChunk.getData();
      idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;
      chunkArray[chunkArrayIdx++] = idxChunk;
    }
    // write data chunk id
    idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataJumboChunk.getId());
    idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataStartOfset);          // offset
    idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, kv.getSerializedSize()); // length
    idxOffset = ByteBufferUtils.putLong(idxBuffer, idxOffset, kv.getSequenceId());     // seqId

    // Jumbo chunks are working only with one cell per chunk, thus always allocate a new jumbo
    // data chunk for next cell
    dataJumboChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP,smallChunkSize);
    dataBuffer = dataJumboChunk.getData();
    dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;
  }

  return new CellChunkMap(CellComparator.getInstance(),chunkArray,0,NUM_OF_CELLS,!asc);
}
 

private CellChunkMap setUpCellChunkMap(boolean asc) {

    // allocate new chunks and use the data chunk to hold the full data of the cells
    // and the index chunk to hold the cell-representations
    Chunk dataChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
    Chunk idxChunk  = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
    // the array of index chunks to be used as a basis for CellChunkMap
    Chunk chunkArray[] = new Chunk[8];  // according to test currently written 8 is way enough
    int chunkArrayIdx = 0;
    chunkArray[chunkArrayIdx++] = idxChunk;

    ByteBuffer idxBuffer = idxChunk.getData();  // the buffers of the chunks
    ByteBuffer dataBuffer = dataChunk.getData();
    int dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;        // offset inside data buffer
    int idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;         // skip the space for chunk ID

    Cell[] cellArray = asc ? ascCells : descCells;

    for (Cell kv: cellArray) {
      // do we have enough space to write the cell data on the data chunk?
      if (dataOffset + kv.getSerializedSize() > chunkCreator.getChunkSize()) {
        // allocate more data chunks if needed
        dataChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
        dataBuffer = dataChunk.getData();
        dataOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;
      }
      int dataStartOfset = dataOffset;
      dataOffset = KeyValueUtil.appendTo(kv, dataBuffer, dataOffset, false); // write deep cell data

      // do we have enough space to write the cell-representation on the index chunk?
      if (idxOffset + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkCreator.getChunkSize()) {
        // allocate more index chunks if needed
        idxChunk = chunkCreator.getChunk(CompactingMemStore.IndexType.CHUNK_MAP);
        idxBuffer = idxChunk.getData();
        idxOffset = ChunkCreator.SIZEOF_CHUNK_HEADER;
        chunkArray[chunkArrayIdx++] = idxChunk;
      }
      idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataChunk.getId()); // write data chunk id
      idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, dataStartOfset);          // offset
      idxOffset = ByteBufferUtils.putInt(idxBuffer, idxOffset, kv.getSerializedSize()); // length
      idxOffset = ByteBufferUtils.putLong(idxBuffer, idxOffset, kv.getSequenceId());     // seqId
    }

    return new CellChunkMap(CellComparator.getInstance(),chunkArray,0,NUM_OF_CELLS,!asc);
  }
 

@Test
public void testDeleteMarkerLongevity() throws Exception {
  try {
    final long now = System.currentTimeMillis();
    EnvironmentEdgeManagerTestHelper.injectEdge(new EnvironmentEdge() {
      @Override
      public long currentTime() {
        return now;
      }
    });
    KeyValue[] kvs = new KeyValue[]{
      /*0*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 100, KeyValue.Type.DeleteFamily), // live
      /*1*/ new KeyValue(Bytes.toBytes("R1"), Bytes.toBytes("cf"), null,
      now - 1000, KeyValue.Type.DeleteFamily), // expired
      /*2*/ create("R1", "cf", "a", now - 50,
      KeyValue.Type.Put, "v3"), // live
      /*3*/ create("R1", "cf", "a", now - 55,
      KeyValue.Type.Delete, "dontcare"), // live
      /*4*/ create("R1", "cf", "a", now - 55,
      KeyValue.Type.Put, "deleted-version v2"), // deleted
      /*5*/ create("R1", "cf", "a", now - 60,
      KeyValue.Type.Put, "v1"), // live
      /*6*/ create("R1", "cf", "a", now - 65,
      KeyValue.Type.Put, "v0"), // max-version reached
      /*7*/ create("R1", "cf", "a",
      now - 100, KeyValue.Type.DeleteColumn, "dont-care"), // max-version
      /*8*/ create("R1", "cf", "b", now - 600,
      KeyValue.Type.DeleteColumn, "dont-care"), //expired
      /*9*/ create("R1", "cf", "b", now - 70,
      KeyValue.Type.Put, "v2"), //live
      /*10*/ create("R1", "cf", "b", now - 750,
      KeyValue.Type.Put, "v1"), //expired
      /*11*/ create("R1", "cf", "c", now - 500,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*12*/ create("R1", "cf", "c", now - 600,
      KeyValue.Type.Put, "v1"), //expired
      /*13*/ create("R1", "cf", "c", now - 1000,
      KeyValue.Type.Delete, "dontcare"), //expired
      /*14*/ create("R1", "cf", "d", now - 60,
      KeyValue.Type.Put, "expired put"), //live
      /*15*/ create("R1", "cf", "d", now - 100,
      KeyValue.Type.Delete, "not-expired delete"), //live
    };
    List<KeyValueScanner> scanners = scanFixture(kvs);
    ScanInfo scanInfo = new ScanInfo(CONF, Bytes.toBytes("cf"),
      0 /* minVersions */,
      2 /* maxVersions */, 500 /* ttl */,
      KeepDeletedCells.FALSE /* keepDeletedCells */,
      HConstants.DEFAULT_BLOCKSIZE /* block size */,
      200, /* timeToPurgeDeletes */
      CellComparator.getInstance(), false);
    try (StoreScanner scanner =
        new StoreScanner(scanInfo, 2, ScanType.COMPACT_DROP_DELETES, scanners)) {
      List<Cell> results = new ArrayList<>();
      results = new ArrayList<>();
      assertEquals(true, scanner.next(results));
      assertEquals(kvs[0], results.get(0));
      assertEquals(kvs[2], results.get(1));
      assertEquals(kvs[3], results.get(2));
      assertEquals(kvs[5], results.get(3));
      assertEquals(kvs[9], results.get(4));
      assertEquals(kvs[14], results.get(5));
      assertEquals(kvs[15], results.get(6));
      assertEquals(7, results.size());
    }
  } finally {
    EnvironmentEdgeManagerTestHelper.reset();
  }
}
 

/**
 * Default constructor. Used for tests.
 */
public DefaultMemStore() {
  this(HBaseConfiguration.create(), CellComparator.getInstance(), null);
}
 

public CollectionBackedScanner(List<Cell> list) {
  this(list, CellComparator.getInstance());
}