org.apache.hadoop.hbase.KeyValueUtil Java Examples

The following examples show how to use org.apache.hadoop.hbase.KeyValueUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: EncodedColumnQualifierCellsListTest.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testFailFastIterator() throws Exception {
    EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
    populateList(list);
    int i = 0;
    Iterator<Cell> itr = list.iterator();
    while (itr.hasNext()) {
        i++;
        try {
            itr.next();
            list.add(KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
            if (i == 2) {
                fail("ConcurrentModificationException should have been thrown as the list is being modified while being iterated through");
            }
        } catch (ConcurrentModificationException expected) {
            assertEquals("Exception should have been thrown when getting the second element",
                2, i);
            break;
        }
    }
}
 
Example #2
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                                 cell.getFamilyArray(), cell.getFamilyOffset(),
                                                 cell.getFamilyLength(), cell.getQualifierArray(),
                                                 cell.getQualifierOffset(), cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #3
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                                 cell.getFamilyArray(), cell.getFamilyOffset(),
                                                 cell.getFamilyLength(), cell.getQualifierArray(),
                                                 cell.getQualifierOffset(), cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #4
Source File: SlicedRowFilter.java    From warp10-platform with Apache License 2.0 6 votes vote down vote up
@Override
public KeyValue getNextKeyHint(KeyValue currentKV) {
  //hintCount++;
  KeyValue hint = null;
      
  if (this.hintOffset >= 0 && this.hintOffset <= this.rangekeys.length - slicesLength) {
    hint = KeyValueUtil.createFirstOnRow(this.rangekeys, this.hintOffset, (short) (this.bounds[1] + 1));
    minRange = (hintOffset / this.slicesLength) / 2;
  } else {
    done = true;
  }

  /*
  byte[] row = currentKV.getRowArray();
  System.out.println("getNextKeyHint " + encodeHex(row, currentKV.getRowOffset(), currentKV.getRowLength()) + " nvalues = " + this.nvalues + " count = " + this.count + " hintOffset = " + hintOffset);
  if (null != hint) {
    row = hint.getRowArray();
    System.out.println("  hint = " + encodeHex(row, hint.getRowOffset(), hint.getRowLength())); 
  } else {
    System.out.println(" hint = null");
  }
  */
  
  return hint;
}
 
Example #5
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                                 cell.getFamilyArray(), cell.getFamilyOffset(),
                                                 cell.getFamilyLength(), cell.getQualifierArray(),
                                                 cell.getQualifierOffset(), cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #6
Source File: IndexedWALEditCodec.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void write(Cell cell) throws IOException {
  //make sure we are open
  checkFlushed();
  
  //write the special marker so we can figure out which kind of kv is it
  int marker = IndexedWALEditCodec.REGULAR_KEY_VALUE_MARKER;
  if (cell instanceof IndexedKeyValue) {
    marker = KeyValueCodec.INDEX_TYPE_LENGTH_MARKER;
  }
  out.write(marker);
  
  //then serialize based on the marker
  if (marker == IndexedWALEditCodec.REGULAR_KEY_VALUE_MARKER) {
    this.compressedKvEncoder.write(cell);
  }
  else{
    KeyValueCodec.write((DataOutput) out, KeyValueUtil.ensureKeyValue(cell));
  }
}
 
Example #7
Source File: TestFilterList.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * When we do a "MUST_PASS_ONE" (a logical 'OR') of the two filters
 * we expect to get the same result as the inclusive stop result.
 * @throws Exception
 */
@Test
public void testFilterListWithInclusiveStopFilterMustPassOne() throws Exception {
  byte[] r1 = Bytes.toBytes("Row1");
  byte[] r11 = Bytes.toBytes("Row11");
  byte[] r2 = Bytes.toBytes("Row2");

  FilterList flist = new FilterList(FilterList.Operator.MUST_PASS_ONE);
  flist.addFilter(new AlwaysNextColFilter());
  flist.addFilter(new InclusiveStopFilter(r1));
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r1));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r1, r1, r1)));
  assertEquals(ReturnCode.INCLUDE, flist.filterCell(new KeyValue(r11, r11, r11)));

  flist.reset();
  flist.filterRowKey(KeyValueUtil.createFirstOnRow(r2));
  assertEquals(ReturnCode.NEXT_COL, flist.filterCell(new KeyValue(r2, r2, r2)));
}
 
Example #8
Source File: CoveredColumnsIndexBuilder.java    From phoenix with Apache License 2.0 6 votes vote down vote up
/**
 * Batch all the {@link KeyValue}s in a {@link Mutation} by timestamp. Updates any
 * {@link KeyValue} with a timestamp == {@link HConstants#LATEST_TIMESTAMP} to the timestamp at
 * the time the method is called.
 * @param m {@link Mutation} from which to extract the {@link KeyValue}s
 * @return the mutation, broken into batches and sorted in ascending order (smallest first)
 */
protected Collection<Batch> createTimestampBatchesFromMutation(Mutation m) {
  Map<Long, Batch> batches = new HashMap<Long, Batch>();
  for (List<Cell> family : m.getFamilyCellMap().values()) {
    List<KeyValue> familyKVs = KeyValueUtil.ensureKeyValues(family);
    createTimestampBatchesFromKeyValues(familyKVs, batches);
  }
  // sort the batches
  List<Batch> sorted = new ArrayList<Batch>(batches.values());
  Collections.sort(sorted, new Comparator<Batch>() {
    @Override
    public int compare(Batch o1, Batch o2) {
      return Longs.compare(o1.getTimestamp(), o2.getTimestamp());
    }
  });
  return sorted;
}
 
Example #9
Source File: TestRandomRowFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Tests basics
 *
 * @throws Exception
 */
@Test
public void testBasics() throws Exception {
  int included = 0;
  int max = 1000000;
  for (int i = 0; i < max; i++) {
    if (!quarterChanceFilter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("row")))) {
      included++;
    }
  }
  // Now let's check if the filter included the right number of rows;
  // since we're dealing with randomness, we must have a include an epsilon
  // tolerance.
  int epsilon = max / 100;
  assertTrue("Roughly 25% should pass the filter", Math.abs(included - max
      / 4) < epsilon);
}
 
Example #10
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                                 cell.getFamilyArray(), cell.getFamilyOffset(),
                                                 cell.getFamilyLength(), cell.getQualifierArray(),
                                                 cell.getQualifierOffset(), cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #11
Source File: CellSkipFilter.java    From phoenix-tephra with Apache License 2.0 6 votes vote down vote up
@Override
public ReturnCode filterKeyValue(Cell cell) throws IOException {
  if (skipCellVersion(cell)) {
    return ReturnCode.NEXT_COL;
  }

  ReturnCode code = filter.filterKeyValue(cell);
  if (code == ReturnCode.NEXT_COL || code == ReturnCode.INCLUDE_AND_NEXT_COL) {
    // only store the reference to the keyvalue if we are returning NEXT_COL or INCLUDE_AND_NEXT_COL
    skipColumn = KeyValueUtil.createFirstOnRow(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(),
                                                 cell.getFamilyArray(), cell.getFamilyOffset(),
                                                 cell.getFamilyLength(), cell.getQualifierArray(),
                                                 cell.getQualifierOffset(), cell.getQualifierLength());
  } else {
    skipColumn = null;
  }
  return code;
}
 
Example #12
Source File: MaxTimestampFilter.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public Cell getNextCellHint(Cell currentKV) {
  // this might be a little excessive right now - better safe than sorry though, so we don't mess
  // with other filters too much.
  KeyValue kv = null;
  try {
      kv = KeyValueUtil.ensureKeyValue(currentKV).clone();
  } catch (CloneNotSupportedException e) {
      // the exception should not happen at all
      throw new IllegalArgumentException(e);
  }
  int offset =kv.getTimestampOffset();
  //set the timestamp in the buffer
  @SuppressWarnings("deprecation")
  byte[] buffer = kv.getBuffer();
  byte[] ts = Bytes.toBytes(this.ts);
  System.arraycopy(ts, 0, buffer, offset, ts.length);

  return kv;
}
 
Example #13
Source File: FilteredKeyValueScanner.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("deprecation")
private boolean seekToNextUnfilteredKeyValue() throws IOException {
    while (true) {
        Cell peeked = delegate.peek();
        // no more key values, so we are done
        if (peeked == null) { return false; }

        // filter the peeked value to see if it should be served
        ReturnCode code = filter.filterKeyValue(peeked);
        switch (code) {
        // included, so we are done
        case INCLUDE:
        case INCLUDE_AND_NEXT_COL:
            return true;
            // not included, so we need to go to the next row
        case SKIP:
        case NEXT_COL:
        case NEXT_ROW:
            delegate.next();
            break;
        // use a seek hint to find out where we should go
        case SEEK_NEXT_USING_HINT:
            delegate.seek(KeyValueUtil.ensureKeyValue(filter.getNextCellHint(peeked)));
        }
    }
}
 
Example #14
Source File: Import.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void setup(Context context) throws IOException {
  cfRenameMap = createCfRenameMap(context.getConfiguration());
  filter = instantiateFilter(context.getConfiguration());
  int reduceNum = context.getNumReduceTasks();
  Configuration conf = context.getConfiguration();
  TableName tableName = TableName.valueOf(context.getConfiguration().get(TABLE_NAME));
  try (Connection conn = ConnectionFactory.createConnection(conf);
      RegionLocator regionLocator = conn.getRegionLocator(tableName)) {
    byte[][] startKeys = regionLocator.getStartKeys();
    if (startKeys.length != reduceNum) {
      throw new IOException("Region split after job initialization");
    }
    CellWritableComparable[] startKeyWraps =
        new CellWritableComparable[startKeys.length - 1];
    for (int i = 1; i < startKeys.length; ++i) {
      startKeyWraps[i - 1] =
          new CellWritableComparable(KeyValueUtil.createFirstOnRow(startKeys[i]));
    }
    CellWritableComparablePartitioner.START_KEYS = startKeyWraps;
  }
}
 
Example #15
Source File: TestVisibilityLabelsReplication.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void prePut(ObserverContext<RegionCoprocessorEnvironment> e, Put m, WALEdit edit,
    Durability durability) throws IOException {
  byte[] attribute = m.getAttribute(NON_VISIBILITY);
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : m.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag((byte) NON_VIS_TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(PrivateCellUtil.getTags(cell).size() + 1);
        tagList.add(tag);
        tagList.addAll(PrivateCellUtil.getTags(cell));
        Cell newcell = PrivateCellUtil.createCell(kv, tagList);
        ((List<Cell>) updatedCells).add(newcell);
      }
    }
    m.getFamilyCellMap().remove(cf);
    // Update the family map
    m.getFamilyCellMap().put(cf, updatedCells);
  }
}
 
Example #16
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected void verifyScanAcrossSnapshot2(KeyValue kv1, KeyValue kv2)
    throws IOException {
  List<KeyValueScanner> memstorescanners = this.memstore.getScanners(mvcc.getReadPoint());
  assertEquals(2, memstorescanners.size());
  final KeyValueScanner scanner0 = memstorescanners.get(0);
  final KeyValueScanner scanner1 = memstorescanners.get(1);
  scanner0.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  scanner1.seek(KeyValueUtil.createFirstOnRow(HConstants.EMPTY_START_ROW));
  Cell n0 = scanner0.next();
  Cell n1 = scanner1.next();
  assertTrue(kv1.equals(n0) || kv1.equals(n1));
  assertTrue(kv2.equals(n0)
      || kv2.equals(n1)
      || kv2.equals(scanner0.next())
      || kv2.equals(scanner1.next()));
  assertNull(scanner0.next());
  assertNull(scanner1.next());
}
 
Example #17
Source File: TestDefaultMemStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected void assertScannerResults(KeyValueScanner scanner, KeyValue[] expected)
    throws IOException {
  scanner.seek(KeyValueUtil.createFirstOnRow(new byte[]{}));
  List<Cell> returned = Lists.newArrayList();

  while (true) {
    Cell next = scanner.next();
    if (next == null) break;
    returned.add(next);
  }

  assertTrue(
      "Got:\n" + Joiner.on("\n").join(returned) +
      "\nExpected:\n" + Joiner.on("\n").join(expected),
      Iterables.elementsEqual(Arrays.asList(expected), returned));
  assertNull(scanner.peek());
}
 
Example #18
Source File: EncodedColumnQualifierCellsListTest.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testContainsAll() throws Exception {
    EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
    populateList(list1);
    EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
    populateList(list2);
    assertTrue(list1.containsAll(list2));
    list2.remove(KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11)));
    assertTrue(list1.containsAll(list2));
    assertFalse(list2.containsAll(list1));
    list2.add(KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13)));
    assertFalse(list1.containsAll(list2));
    assertFalse(list2.containsAll(list1));
    List<Cell> arrayList = new ArrayList<>();
    populateList(arrayList);
    assertTrue(list1.containsAll(arrayList));
}
 
Example #19
Source File: StoreScanner.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void shipped() throws IOException {
  if (prevCell != null) {
    // Do the copy here so that in case the prevCell ref is pointing to the previous
    // blocks we can safely release those blocks.
    // This applies to blocks that are got from Bucket cache, L1 cache and the blocks
    // fetched from HDFS. Copying this would ensure that we let go the references to these
    // blocks so that they can be GCed safely(in case of bucket cache)
    prevCell = KeyValueUtil.toNewKeyCell(this.prevCell);
  }
  matcher.beforeShipped();
  // There wont be further fetch of Cells from these scanners. Just close.
  clearAndClose(scannersForDelayedClose);
  if (this.heap != null) {
    this.heap.shipped();
    // When switching from pread to stream, we will open a new scanner for each store file, but
    // the old scanner may still track the HFileBlocks we have scanned but not sent back to client
    // yet. If we close the scanner immediately then the HFileBlocks may be messed up by others
    // before we serialize and send it back to client. The HFileBlocks will be released in shipped
    // method, so we here will also open new scanners and close old scanners in shipped method.
    // See HBASE-18055 for more details.
    trySwitchToStreamRead();
  }
}
 
Example #20
Source File: TestCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void testDropDeletes(byte[] from, byte[] to, byte[][] rows, MatchCode... expected)
    throws IOException {
  long now = EnvironmentEdgeManager.currentTime();
  // Set time to purge deletes to negative value to avoid it ever happening.
  ScanInfo scanInfo = new ScanInfo(this.conf, fam2, 0, 1, ttl, KeepDeletedCells.FALSE,
      HConstants.DEFAULT_BLOCKSIZE, -1L, rowComparator, false);

  CompactionScanQueryMatcher qm = CompactionScanQueryMatcher.create(scanInfo,
    ScanType.COMPACT_RETAIN_DELETES, Long.MAX_VALUE, HConstants.OLDEST_TIMESTAMP,
    HConstants.OLDEST_TIMESTAMP, now, from, to, null);
  List<ScanQueryMatcher.MatchCode> actual = new ArrayList<>(rows.length);
  byte[] prevRow = null;
  for (byte[] row : rows) {
    if (prevRow == null || !Bytes.equals(prevRow, row)) {
      qm.setToNewRow(KeyValueUtil.createFirstOnRow(row));
      prevRow = row;
    }
    actual.add(qm.match(new KeyValue(row, fam2, null, now, Type.Delete)));
  }

  assertEquals(expected.length, actual.size());
  for (int i = 0; i < expected.length; i++) {
    LOG.debug("expected " + expected[i] + ", actual " + actual.get(i));
    assertEquals(expected[i], actual.get(i));
  }
}
 
Example #21
Source File: NoneEncoder.java    From hbase with Apache License 2.0 6 votes vote down vote up
public int write(Cell cell) throws IOException {
  // We write tags seperately because though there is no tag in KV
  // if the hfilecontext says include tags we need the tags length to be
  // written
  int size = KeyValueUtil.oswrite(cell, out, false);
  // Write the additional tag into the stream
  if (encodingCtx.getHFileContext().isIncludesTags()) {
    int tagsLength = cell.getTagsLength();
    out.writeShort(tagsLength);
    if (tagsLength > 0) {
      PrivateCellUtil.writeTags(out, cell, tagsLength);
    }
    size += tagsLength + KeyValue.TAGS_LENGTH_SIZE;
  }
  if (encodingCtx.getHFileContext().isIncludesMvcc()) {
    WritableUtils.writeVLong(out, cell.getSequenceId());
    size += WritableUtils.getVIntSize(cell.getSequenceId());
  }
  return size;
}
 
Example #22
Source File: TestCompoundBloomFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateKey() {
  byte[] row = Bytes.toBytes("myRow");
  byte[] qualifier = Bytes.toBytes("myQualifier");
  // Mimic what Storefile.createBloomKeyValue() does
  byte[] rowKey = KeyValueUtil.createFirstOnRow(row, 0, row.length, new byte[0], 0, 0, row, 0, 0).getKey();
  byte[] rowColKey = KeyValueUtil.createFirstOnRow(row, 0, row.length,
      new byte[0], 0, 0, qualifier, 0, qualifier.length).getKey();
  KeyValue rowKV = KeyValueUtil.createKeyValueFromKey(rowKey);
  KeyValue rowColKV = KeyValueUtil.createKeyValueFromKey(rowColKey);
  assertEquals(rowKV.getTimestamp(), rowColKV.getTimestamp());
  assertEquals(Bytes.toStringBinary(rowKV.getRowArray(), rowKV.getRowOffset(),
    rowKV.getRowLength()), Bytes.toStringBinary(rowColKV.getRowArray(), rowColKV.getRowOffset(),
    rowColKV.getRowLength()));
  assertEquals(0, rowKV.getQualifierLength());
}
 
Example #23
Source File: TestReversibleScanners.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void internalTestSeekAndNextForReversibleKeyValueHeap(
    ReversedKeyValueHeap kvHeap, int startRowNum) throws IOException {
  // Test next and seek
  for (int i = startRowNum; i >= 0; i--) {
    if (i % 2 == 1 && i - 2 >= 0) {
      i = i - 2;
      kvHeap.seekToPreviousRow(KeyValueUtil.createFirstOnRow(ROWS[i + 1]));
    }
    for (int j = 0; j < QUALSIZE; j++) {
      if (j % 2 == 1 && (j + 1) < QUALSIZE) {
        j = j + 1;
        kvHeap.backwardSeek(makeKV(i, j));
      }
      assertEquals(makeKV(i, j), kvHeap.peek());
      kvHeap.next();
    }
  }
  assertEquals(null, kvHeap.peek());
}
 
Example #24
Source File: TestMultiRowRangeFilter.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testOutOfOrderScannerNextException() throws Exception {
  MultiRowRangeFilter filter = new MultiRowRangeFilter(Arrays.asList(
          new MultiRowRangeFilter.RowRange(Bytes.toBytes("b"), true, Bytes.toBytes("c"), true),
          new MultiRowRangeFilter.RowRange(Bytes.toBytes("d"), true, Bytes.toBytes("e"), true)
  ));
  filter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("a")));
  assertEquals(Filter.ReturnCode.SEEK_NEXT_USING_HINT, filter.filterCell(null));
  filter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("b")));
  assertEquals(Filter.ReturnCode.INCLUDE, filter.filterCell(null));
  filter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("c")));
  assertEquals(Filter.ReturnCode.INCLUDE, filter.filterCell(null));
  filter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("d")));
  assertEquals(Filter.ReturnCode.INCLUDE, filter.filterCell(null));
  filter.filterRowKey(KeyValueUtil.createFirstOnRow(Bytes.toBytes("e")));
  assertEquals(Filter.ReturnCode.INCLUDE, filter.filterCell(null));
}
 
Example #25
Source File: TestStoreFileScannerWithTagCompression.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testReseek() throws Exception {
  // write the file
  Path f = new Path(ROOT_DIR, "testReseek");
  HFileContext meta = new HFileContextBuilder().withBlockSize(8 * 1024).withIncludesTags(true)
      .withCompressTags(true).withDataBlockEncoding(DataBlockEncoding.PREFIX).build();
  // Make a store file and write data to it.
  StoreFileWriter writer = new StoreFileWriter.Builder(conf, cacheConf, fs).withFilePath(f)
      .withFileContext(meta).build();

  writeStoreFile(writer);
  writer.close();

  ReaderContext context = new ReaderContextBuilder().withFileSystemAndPath(fs, f).build();
  HFileInfo fileInfo = new HFileInfo(context, conf);
  StoreFileReader reader =
      new StoreFileReader(context, fileInfo, cacheConf, new AtomicInteger(0), conf);
  fileInfo.initMetaAndIndex(reader.getHFileReader());
  StoreFileScanner s = reader.getStoreFileScanner(false, false, false, 0, 0, false);
  try {
    // Now do reseek with empty KV to position to the beginning of the file
    KeyValue k = KeyValueUtil.createFirstOnRow(Bytes.toBytes("k2"));
    s.reseek(k);
    Cell kv = s.next();
    kv = s.next();
    kv = s.next();
    byte[] key5 = Bytes.toBytes("k5");
    assertTrue(Bytes.equals(key5, 0, key5.length, kv.getRowArray(), kv.getRowOffset(),
        kv.getRowLength()));
    List<Tag> tags = PrivateCellUtil.getTags(kv);
    assertEquals(1, tags.size());
    assertEquals("tag3", Bytes.toString(Tag.cloneValue(tags.get(0))));
  } finally {
    s.close();
  }
}
 
Example #26
Source File: TTable.java    From phoenix-omid with Apache License 2.0 5 votes vote down vote up
private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 
Example #27
Source File: HFilePrettyPrinter.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void collect(Cell cell) {
  valLen.update(cell.getValueLength());
  if (prevCell != null &&
      CellComparator.getInstance().compareRows(prevCell, cell) != 0) {
    // new row
    collectRow();
  }
  curRowBytes += cell.getSerializedSize();
  curRowKeyLength = KeyValueUtil.keyLength(cell);
  curRowCols++;
  prevCell = cell;
}
 
Example #28
Source File: HFileWriterImpl.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void beforeShipped() throws IOException {
  this.blockWriter.beforeShipped();
  // Add clone methods for every cell
  if (this.lastCell != null) {
    this.lastCell = KeyValueUtil.toNewKeyCell(this.lastCell);
  }
  if (this.firstCellInBlock != null) {
    this.firstCellInBlock = KeyValueUtil.toNewKeyCell(this.firstCellInBlock);
  }
  if (this.lastCellOfPreviousBlock != null) {
    this.lastCellOfPreviousBlock = KeyValueUtil.toNewKeyCell(this.lastCellOfPreviousBlock);
  }
}
 
Example #29
Source File: EncodedColumnQualifierCellsListTest.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private void populateListAndArray(List<Cell> list, Cell[] cells) {
    // add elements in reserved range
    list.add(cells[0] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
    list.add(cells[1] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(5)));
    list.add(cells[2] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(10)));

    // add elements in qualifier range
    list.add(cells[6] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(16)));
    list.add(cells[4] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12)));
    list.add(cells[5] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(14)));
    list.add(cells[3] = KeyValueUtil.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11)));
}
 
Example #30
Source File: TestReplicationWithTags.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put,
    final WALEdit edit, final Durability durability) throws IOException {
  byte[] attribute = put.getAttribute("visibility");
  byte[] cf = null;
  List<Cell> updatedCells = new ArrayList<>();
  if (attribute != null) {
    for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
      for (Cell cell : edits) {
        KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
        if (cf == null) {
          cf = CellUtil.cloneFamily(kv);
        }
        Tag tag = new ArrayBackedTag(TAG_TYPE, attribute);
        List<Tag> tagList = new ArrayList<>(1);
        tagList.add(tag);

        KeyValue newKV = new KeyValue(CellUtil.cloneRow(kv), 0, kv.getRowLength(),
            CellUtil.cloneFamily(kv), 0, kv.getFamilyLength(), CellUtil.cloneQualifier(kv), 0,
            kv.getQualifierLength(), kv.getTimestamp(),
            KeyValue.Type.codeToType(kv.getTypeByte()), CellUtil.cloneValue(kv), 0,
            kv.getValueLength(), tagList);
        ((List<Cell>) updatedCells).add(newKV);
      }
    }
    put.getFamilyCellMap().remove(cf);
    // Update the family map
    put.getFamilyCellMap().put(cf, updatedCells);
  }
}