Java Code Examples for org.apache.hadoop.hbase.Cell#getTypeByte()

The following examples show how to use org.apache.hadoop.hbase.Cell#getTypeByte() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MetaDataEndpointImpl.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private PTable buildDeletedTable(byte[] key, ImmutableBytesPtr cacheKey, HRegion region,
    long clientTimeStamp) throws IOException {
    if (clientTimeStamp == HConstants.LATEST_TIMESTAMP) {
        return null;
    }

    Scan scan = MetaDataUtil.newTableRowsScan(key, clientTimeStamp, HConstants.LATEST_TIMESTAMP);
    scan.setFilter(new FirstKeyOnlyFilter());
    scan.setRaw(true);
    List<Cell> results = Lists.<Cell> newArrayList();
    try (RegionScanner scanner = region.getScanner(scan);) {
      scanner.next(results);
    }
    // HBase ignores the time range on a raw scan (HBASE-7362)
    if (!results.isEmpty() && results.get(0).getTimestamp() > clientTimeStamp) {
        Cell kv = results.get(0);
        if (kv.getTypeByte() == Type.Delete.getCode()) {
            Cache<ImmutableBytesPtr, PTable> metaDataCache =
                    GlobalCache.getInstance(this.env).getMetaDataCache();
            PTable table = newDeletedTableMarker(kv.getTimestamp());
            metaDataCache.put(cacheKey, table);
            return table;
        }
    }
    return null;
}
 
Example 2
Source File: StoreFileReader.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * A method for checking Bloom filters. Called directly from
 * StoreFileScanner in case of a multi-column query.
 *
 * @param cell
 *          the cell to check if present in BloomFilter
 * @return True if passes
 */
public boolean passesGeneralRowColBloomFilter(Cell cell) {
  BloomFilter bloomFilter = this.generalBloomFilter;
  if (bloomFilter == null) {
    return true;
  }
  // Used in ROW_COL bloom
  Cell kvKey = null;
  // Already if the incoming key is a fake rowcol key then use it as it is
  if (cell.getTypeByte() == KeyValue.Type.Maximum.getCode() && cell.getFamilyLength() == 0) {
    kvKey = cell;
  } else {
    kvKey = PrivateCellUtil.createFirstOnRowCol(cell);
  }
  return checkGeneralBloomFilter(null, kvKey, bloomFilter);
}
 
Example 3
Source File: NewVersionBehaviorTracker.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void add(Cell cell) {
  prepare(cell);
  byte type = cell.getTypeByte();
  switch (Type.codeToType(type)) {
  // By the order of seen. We put null cq at first.
  case DeleteFamily: // Delete all versions of all columns of the specified family
    delFamMap.put(cell.getSequenceId(),
        new DeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId()));
    break;
  case DeleteFamilyVersion: // Delete all columns of the specified family and specified version
    delFamMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell);
    break;

  // These two kinds of markers are mix with Puts.
  case DeleteColumn: // Delete all versions of the specified column
    delColMap.put(cell.getSequenceId(),
        new DeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId()));
    break;
  case Delete: // Delete the specified version of the specified column.
    delColMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell);
    break;
  default:
    throw new AssertionError("Unknown delete marker type for " + cell);
  }
}
 
Example 4
Source File: NormalUserScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public MatchCode match(Cell cell) throws IOException {
  if (filter != null && filter.filterAllRemaining()) {
    return MatchCode.DONE_SCAN;
  }
  MatchCode returnCode = preCheck(cell);
  if (returnCode != null) {
    return returnCode;
  }
  long timestamp = cell.getTimestamp();
  byte typeByte = cell.getTypeByte();
  if (PrivateCellUtil.isDelete(typeByte)) {
    boolean includeDeleteMarker = seePastDeleteMarkers ? tr.withinTimeRange(timestamp)
        : tr.withinOrAfterTimeRange(timestamp);
    if (includeDeleteMarker) {
      this.deletes.add(cell);
    }
    return MatchCode.SKIP;
  }
  returnCode = checkDeleted(deletes, cell);
  if (returnCode != null) {
    return returnCode;
  }
  return matchColumn(cell, timestamp, typeByte);
}
 
Example 5
Source File: MinorCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public MatchCode match(Cell cell) throws IOException {
  MatchCode returnCode = preCheck(cell);
  if (returnCode != null) {
    return returnCode;
  }
  long mvccVersion = cell.getSequenceId();
  byte typeByte = cell.getTypeByte();
  if (PrivateCellUtil.isDelete(typeByte)) {
    if (mvccVersion > maxReadPointToTrackVersions) {
      // we should not use this delete marker to mask any cell yet.
      return MatchCode.INCLUDE;
    }
    trackDelete(cell);
    return MatchCode.INCLUDE;
  }
  returnCode = checkDeleted(deletes, cell);
  if (returnCode != null) {
    return returnCode;
  }
  // Skip checking column since we do not remove column during compaction.
  return columns.checkVersions(cell, cell.getTimestamp(), typeByte,
    mvccVersion > maxReadPointToTrackVersions);
}
 
Example 6
Source File: IndexMaintainer.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private DeleteType getDeleteTypeOrNull(Collection<? extends Cell> pendingUpdates, int nCFs) {
      int nDeleteCF = 0;
      int nDeleteVersionCF = 0;
      for (Cell kv : pendingUpdates) {
      	if (kv.getTypeByte() == KeyValue.Type.DeleteFamilyVersion.getCode()) {
              nDeleteVersionCF++;
          }
      	else if (kv.getTypeByte() == KeyValue.Type.DeleteFamily.getCode()
      			// Since we don't include the index rows in the change set for txn tables, we need to detect row deletes that have transformed by TransactionProcessor
      	        || TransactionUtil.isDeleteFamily(kv)) {
      	    nDeleteCF++;
      	}
      }
      // This is what a delete looks like on the server side for mutable indexing...
      // Should all be one or the other for DeleteFamily versus DeleteFamilyVersion, but just in case not
      DeleteType deleteType = null; 
      if (nDeleteVersionCF > 0 && nDeleteVersionCF >= nCFs) {
      	deleteType = DeleteType.SINGLE_VERSION;
      } else {
	int nDelete = nDeleteCF + nDeleteVersionCF;
	if (nDelete>0 && nDelete >= nCFs) {
		deleteType = DeleteType.ALL_VERSIONS;
	}
}
      return deleteType;
  }
 
Example 7
Source File: VisibilityScanDeleteTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void add(Cell delCell) {
  //Cannot call super.add because need to find if the delete needs to be considered
  long timestamp = delCell.getTimestamp();
  byte type = delCell.getTypeByte();
  if (type == KeyValue.Type.DeleteFamily.getCode()) {
    hasFamilyStamp = true;
    boolean hasVisTag = extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamily);
    if (!hasVisTag && timestamp > familyStamp) {
      familyStamp = timestamp;
    }
    return;
  } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
    familyVersionStamps.add(timestamp);
    extractDeleteCellVisTags(delCell, KeyValue.Type.DeleteFamilyVersion);
    return;
  }
  // new column, or more general delete type
  if (deleteCell != null) {
    if (!(CellUtil.matchingQualifier(delCell, deleteCell))) {
      // A case where there are deletes for a column qualifier but there are
      // no corresponding puts for them. Rare case.
      visibilityTagsDeleteColumns = null;
      visiblityTagsDeleteColumnVersion = null;
    } else if (type == KeyValue.Type.Delete.getCode() && (deleteTimestamp != timestamp)) {
      // there is a timestamp change which means we could clear the list
      // when ts is same and the vis tags are different we need to collect
      // them all. Interesting part is that in the normal case of puts if
      // there are 2 cells with same ts and diff vis tags only one of them is
      // returned. Handling with a single List<Tag> would mean that only one
      // of the cell would be considered. Doing this as a precaution.
      // Rare cases.
      visiblityTagsDeleteColumnVersion = null;
    }
  }
  deleteCell = delCell;
  deleteType = type;
  deleteTimestamp = timestamp;
  extractDeleteCellVisTags(delCell, KeyValue.Type.codeToType(type));
}
 
Example 8
Source File: VisibilityNewVersionBehaivorTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void add(Cell cell) {
  prepare(cell);
  byte type = cell.getTypeByte();
  switch (KeyValue.Type.codeToType(type)) {
  // By the order of seen. We put null cq at first.
  case DeleteFamily: // Delete all versions of all columns of the specified family
    delFamMap.put(cell.getSequenceId(),
        new VisibilityDeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId(),
            new TagInfo(cell)));
    break;
  case DeleteFamilyVersion: // Delete all columns of the specified family and specified version
    delFamMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell);
    break;

  // These two kinds of markers are mix with Puts.
  case DeleteColumn: // Delete all versions of the specified column
    delColMap.put(cell.getSequenceId(),
        new VisibilityDeleteVersionsNode(cell.getTimestamp(), cell.getSequenceId(),
            new TagInfo(cell)));
    break;
  case Delete: // Delete the specified version of the specified column.
    delColMap.ceilingEntry(cell.getSequenceId()).getValue().addVersionDelete(cell);
    break;
  default:
    throw new AssertionError("Unknown delete marker type for " + cell);
  }
}
 
Example 9
Source File: StripeCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public MatchCode match(Cell cell) throws IOException {
  MatchCode returnCode = preCheck(cell);
  if (returnCode != null) {
    return returnCode;
  }
  long mvccVersion = cell.getSequenceId();
  byte typeByte = cell.getTypeByte();
  if (PrivateCellUtil.isDelete(typeByte)) {
    if (mvccVersion > maxReadPointToTrackVersions) {
      return MatchCode.INCLUDE;
    }
    trackDelete(cell);
    if (dropDeletesInOutput == DropDeletesInOutput.IN) {
      // here we are running like major compaction
      trackDelete(cell);
      returnCode = tryDropDelete(cell);
      if (returnCode != null) {
        return returnCode;
      }
    } else {
      return MatchCode.INCLUDE;
    }
  } else {
    returnCode = checkDeleted(deletes, cell);
    if (returnCode != null) {
      return returnCode;
    }
  }
  // Skip checking column since we do not remove column during compaction.
  return columns.checkVersions(cell, cell.getTimestamp(), typeByte,
    mvccVersion > maxReadPointToTrackVersions);
}
 
Example 10
Source File: IndexMaintainer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private boolean hasIndexedColumnChanged(ValueGetter oldState, Collection<? extends Cell> pendingUpdates, long ts) throws IOException {
    if (pendingUpdates.isEmpty()) {
        return false;
    }
    Map<ColumnReference,Cell> newState = Maps.newHashMapWithExpectedSize(pendingUpdates.size()); 
    for (Cell kv : pendingUpdates) {
        newState.put(new ColumnReference(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv)), kv);
    }
    for (ColumnReference ref : indexedColumns) {
    	Cell newValue = newState.get(ref);
    	if (newValue != null) { // Indexed column has potentially changed
    	    ImmutableBytesWritable oldValue = oldState.getLatestValue(ref, ts);
            boolean newValueSetAsNull = (newValue.getTypeByte() == Type.DeleteColumn.getCode() || newValue.getTypeByte() == Type.Delete.getCode() || CellUtil.matchingValue(newValue, HConstants.EMPTY_BYTE_ARRAY));
    		boolean oldValueSetAsNull = oldValue == null || oldValue.getLength() == 0;
    		//If the new column value has to be set as null and the older value is null too,
    		//then just skip to the next indexed column.
    		if (newValueSetAsNull && oldValueSetAsNull) {
    			continue;
    		}
    		if (oldValueSetAsNull || newValueSetAsNull) {
    			return true;
    		}
    		// If the old value is different than the new value, the index row needs to be deleted
    		if (Bytes.compareTo(oldValue.get(), oldValue.getOffset(), oldValue.getLength(), 
    				newValue.getValueArray(), newValue.getValueOffset(), newValue.getValueLength()) != 0) {
    			return true;
    		}
    	}
    }
    return false;
}
 
Example 11
Source File: ScanDeleteTracker.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Add the specified Cell to the list of deletes to check against for this row operation.
 * <p>
 * This is called when a Delete is encountered.
 * @param cell - the delete cell
 */
@Override
public void add(Cell cell) {
  long timestamp = cell.getTimestamp();
  byte type = cell.getTypeByte();
  if (!hasFamilyStamp || timestamp > familyStamp) {
    if (type == KeyValue.Type.DeleteFamily.getCode()) {
      hasFamilyStamp = true;
      familyStamp = timestamp;
      return;
    } else if (type == KeyValue.Type.DeleteFamilyVersion.getCode()) {
      familyVersionStamps.add(timestamp);
      return;
    }

    if (deleteCell != null && type < deleteType) {
      // same column, so ignore less specific delete
      if (CellUtil.matchingQualifier(cell, deleteCell)) {
        return;
      }
    }
    // new column, or more general delete type
    deleteCell = cell;
    deleteType = type;
    deleteTimestamp = timestamp;
  }
  // missing else is never called.
}
 
Example 12
Source File: StoreFileWriter.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Record the earlest Put timestamp.
 *
 * If the timeRangeTracker is not set,
 * update TimeRangeTracker to include the timestamp of this key
 */
public void trackTimestamps(final Cell cell) {
  if (KeyValue.Type.Put.getCode() == cell.getTypeByte()) {
    earliestPutTs = Math.min(earliestPutTs, cell.getTimestamp());
  }
  timeRangeTracker.includeTimestamp(cell);
}
 
Example 13
Source File: BufferedDataBlockEncoder.java    From hbase with Apache License 2.0 5 votes vote down vote up
private int compareTypeBytes(Cell key, Cell right) {
  if (key.getFamilyLength() + key.getQualifierLength() == 0
      && key.getTypeByte() == Type.Minimum.getCode()) {
    // left is "bigger", i.e. it appears later in the sorted order
    return 1;
  }
  if (right.getFamilyLength() + right.getQualifierLength() == 0
      && right.getTypeByte() == Type.Minimum.getCode()) {
    return -1;
  }
  return 0;
}
 
Example 14
Source File: ReplicationSink.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * @return True if we have crossed over onto a new row or type
 */
private boolean isNewRowOrType(final Cell previousCell, final Cell cell) {
  return previousCell == null || previousCell.getTypeByte() != cell.getTypeByte() ||
      !CellUtil.matchingRows(previousCell, cell);
}
 
Example 15
Source File: MutableSegment.java    From hbase with Apache License 2.0 4 votes vote down vote up
public void upsert(Cell cell, long readpoint, MemStoreSizing memStoreSizing,
    boolean sizeAddedPreOperation) {
  internalAdd(cell, false, memStoreSizing, sizeAddedPreOperation);

  // Get the Cells for the row/family/qualifier regardless of timestamp.
  // For this case we want to clean up any other puts
  Cell firstCell = PrivateCellUtil.createFirstOnRowColTS(cell, HConstants.LATEST_TIMESTAMP);
  SortedSet<Cell> ss = this.tailSet(firstCell);
  Iterator<Cell> it = ss.iterator();
  // versions visible to oldest scanner
  int versionsVisible = 0;
  while (it.hasNext()) {
    Cell cur = it.next();

    if (cell == cur) {
      // ignore the one just put in
      continue;
    }
    // check that this is the row and column we are interested in, otherwise bail
    if (CellUtil.matchingRows(cell, cur) && CellUtil.matchingQualifier(cell, cur)) {
      // only remove Puts that concurrent scanners cannot possibly see
      if (cur.getTypeByte() == KeyValue.Type.Put.getCode() && cur.getSequenceId() <= readpoint) {
        if (versionsVisible >= 1) {
          // if we get here we have seen at least one version visible to the oldest scanner,
          // which means we can prove that no scanner will see this version

          // false means there was a change, so give us the size.
          // TODO when the removed cell ie.'cur' having its data in MSLAB, we can not release that
          // area. Only the Cell object as such going way. We need to consider cellLen to be
          // decreased there as 0 only. Just keeping it as existing code now. We need to know the
          // removed cell is from MSLAB or not. Will do once HBASE-16438 is in
          int cellLen = getCellLength(cur);
          long heapSize = heapSizeChange(cur, true);
          long offHeapSize = offHeapSizeChange(cur, true);
          incMemStoreSize(-cellLen, -heapSize, -offHeapSize, -1);
          if (memStoreSizing != null) {
            memStoreSizing.decMemStoreSize(cellLen, heapSize, offHeapSize, 1);
          }
          it.remove();
        } else {
          versionsVisible++;
        }
      }
    } else {
      // past the row or column, done
      break;
    }
  }
}
 
Example 16
Source File: MajorCompactionScanQueryMatcher.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public MatchCode match(Cell cell) throws IOException {
  MatchCode returnCode = preCheck(cell);
  if (returnCode != null) {
    return returnCode;
  }
  long timestamp = cell.getTimestamp();
  long mvccVersion = cell.getSequenceId();
  byte typeByte = cell.getTypeByte();

  // The delete logic is pretty complicated now.
  // This is corroborated by the following:
  // 1. The store might be instructed to keep deleted rows around.
  // 2. A scan can optionally see past a delete marker now.
  // 3. If deleted rows are kept, we have to find out when we can
  // remove the delete markers.
  // 4. Family delete markers are always first (regardless of their TS)
  // 5. Delete markers should not be counted as version
  // 6. Delete markers affect puts of the *same* TS
  // 7. Delete marker need to be version counted together with puts
  // they affect
  //
  if (PrivateCellUtil.isDelete(typeByte)) {
    if (mvccVersion > maxReadPointToTrackVersions) {
      // We can not drop this delete marker yet, and also we should not use this delete marker to
      // mask any cell yet.
      return MatchCode.INCLUDE;
    }
    trackDelete(cell);
    returnCode = tryDropDelete(cell);
    if (returnCode != null) {
      return returnCode;
    }
  } else {
    returnCode = checkDeleted(deletes, cell);
    if (returnCode != null) {
      return returnCode;
    }
  }
  // Skip checking column since we do not remove column during compaction.
  return columns.checkVersions(cell, timestamp, typeByte,
    mvccVersion > maxReadPointToTrackVersions);
}
 
Example 17
Source File: DiffKeyDeltaEncoder.java    From hbase with Apache License 2.0 4 votes vote down vote up
private int compressSingleKeyValue(DataOutputStream out, Cell cell, Cell prevCell)
    throws IOException {
  int flag = 0; // Do not use more bits that can fit into a byte
  int kLength = KeyValueUtil.keyLength(cell);
  int vLength = cell.getValueLength();

  long timestamp;
  long diffTimestamp = 0;
  int diffTimestampFitsInBytes = 0;
  int timestampFitsInBytes;
  int commonPrefix = 0;

  if (prevCell == null) {
    timestamp = cell.getTimestamp();
    if (timestamp < 0) {
      flag |= FLAG_TIMESTAMP_SIGN;
      timestamp = -timestamp;
    }
    timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
    flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
    // put column family
    byte familyLength = cell.getFamilyLength();
    out.write(familyLength);
    PrivateCellUtil.writeFamily(out, cell, familyLength);
  } else {
    // Finding common prefix
    int preKeyLength = KeyValueUtil.keyLength(prevCell);
    commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(cell, prevCell, true, false);
    if (kLength == preKeyLength) {
      flag |= FLAG_SAME_KEY_LENGTH;
    }
    if (vLength == prevCell.getValueLength()) {
      flag |= FLAG_SAME_VALUE_LENGTH;
    }
    if (cell.getTypeByte() == prevCell.getTypeByte()) {
      flag |= FLAG_SAME_TYPE;
    }
    // don't compress timestamp and type using prefix encode timestamp
    timestamp = cell.getTimestamp();
    diffTimestamp = prevCell.getTimestamp() - timestamp;
    boolean negativeTimestamp = timestamp < 0;
    if (negativeTimestamp) {
      timestamp = -timestamp;
    }
    timestampFitsInBytes = ByteBufferUtils.longFitsIn(timestamp);
    boolean minusDiffTimestamp = diffTimestamp < 0;
    if (minusDiffTimestamp) {
      diffTimestamp = -diffTimestamp;
    }
    diffTimestampFitsInBytes = ByteBufferUtils.longFitsIn(diffTimestamp);
    if (diffTimestampFitsInBytes < timestampFitsInBytes) {
      flag |= (diffTimestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
      flag |= FLAG_TIMESTAMP_IS_DIFF;
      if (minusDiffTimestamp) {
        flag |= FLAG_TIMESTAMP_SIGN;
      }
    } else {
      flag |= (timestampFitsInBytes - 1) << SHIFT_TIMESTAMP_LENGTH;
      if (negativeTimestamp) {
        flag |= FLAG_TIMESTAMP_SIGN;
      }
    }
  }
  out.write(flag);
  if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
    ByteBufferUtils.putCompressedInt(out, kLength);
  }
  if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
    ByteBufferUtils.putCompressedInt(out, vLength);
  }
  ByteBufferUtils.putCompressedInt(out, commonPrefix);
  short rLen = cell.getRowLength();
  if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
    // Previous and current rows are different. Copy the differing part of
    // the row, skip the column family, and copy the qualifier.
    PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
    PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
  } else {
    // The common part includes the whole row. As the column family is the
    // same across the whole file, it will automatically be included in the
    // common prefix, so we need not special-case it here.
    // What we write here is the non common part of the qualifier
    int commonQualPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
        - (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
    PrivateCellUtil.writeQualifierSkippingBytes(out, cell, cell.getQualifierLength(),
      commonQualPrefix);
  }
  if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) {
    ByteBufferUtils.putLong(out, timestamp, timestampFitsInBytes);
  } else {
    ByteBufferUtils.putLong(out, diffTimestamp, diffTimestampFitsInBytes);
  }

  if ((flag & FLAG_SAME_TYPE) == 0) {
    out.write(cell.getTypeByte());
  }
  PrivateCellUtil.writeValue(out, cell, vLength);
  return kLength + vLength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}
 
Example 18
Source File: FastDiffDeltaEncoder.java    From hbase with Apache License 2.0 4 votes vote down vote up
private int compressSingleKeyValue(DataOutputStream out, Cell cell, Cell prevCell)
    throws IOException {
  int flag = 0; // Do not use more bits than will fit into a byte
  int kLength = KeyValueUtil.keyLength(cell);
  int vLength = cell.getValueLength();

  if (prevCell == null) {
    // copy the key, there is no common prefix with none
    out.write(flag);
    ByteBufferUtils.putCompressedInt(out, kLength);
    ByteBufferUtils.putCompressedInt(out, vLength);
    ByteBufferUtils.putCompressedInt(out, 0);
    PrivateCellUtil.writeFlatKey(cell, (DataOutput)out);
    // Write the value part
    PrivateCellUtil.writeValue(out, cell, cell.getValueLength());
  } else {
    int preKeyLength = KeyValueUtil.keyLength(prevCell);
    int preValLength = prevCell.getValueLength();
    // find a common prefix and skip it
    int commonPrefix = PrivateCellUtil.findCommonPrefixInFlatKey(cell, prevCell, true, false);

    if (kLength == preKeyLength) {
      flag |= FLAG_SAME_KEY_LENGTH;
    }
    if (vLength == prevCell.getValueLength()) {
      flag |= FLAG_SAME_VALUE_LENGTH;
    }
    if (cell.getTypeByte() == prevCell.getTypeByte()) {
      flag |= FLAG_SAME_TYPE;
    }

    byte[] curTsBuf = Bytes.toBytes(cell.getTimestamp());
    int commonTimestampPrefix = findCommonTimestampPrefix(curTsBuf,
        Bytes.toBytes(prevCell.getTimestamp()));

    flag |= commonTimestampPrefix << SHIFT_TIMESTAMP_LENGTH;

    // Check if current and previous values are the same. Compare value
    // length first as an optimization.
    if (vLength == preValLength
        && PrivateCellUtil.matchingValue(cell, prevCell, vLength, preValLength)) {
      flag |= FLAG_SAME_VALUE;
    }

    out.write(flag);
    if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
      ByteBufferUtils.putCompressedInt(out, kLength);
    }
    if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
      ByteBufferUtils.putCompressedInt(out, vLength);
    }
    ByteBufferUtils.putCompressedInt(out, commonPrefix);
    short rLen = cell.getRowLength();
    if (commonPrefix < rLen + KeyValue.ROW_LENGTH_SIZE) {
      // Previous and current rows are different. Copy the differing part of
      // the row, skip the column family, and copy the qualifier.
      PrivateCellUtil.writeRowKeyExcludingCommon(cell, rLen, commonPrefix, out);
      PrivateCellUtil.writeQualifier(out, cell, cell.getQualifierLength());
    } else {
      // The common part includes the whole row. As the column family is the
      // same across the whole file, it will automatically be included in the
      // common prefix, so we need not special-case it here.
      // What we write here is the non common part of the qualifier
      int commonQualPrefix = commonPrefix - (rLen + KeyValue.ROW_LENGTH_SIZE)
          - (cell.getFamilyLength() + KeyValue.FAMILY_LENGTH_SIZE);
      PrivateCellUtil.writeQualifierSkippingBytes(out, cell, cell.getQualifierLength(),
        commonQualPrefix);
    }
    // Write non common ts part
    out.write(curTsBuf, commonTimestampPrefix, KeyValue.TIMESTAMP_SIZE - commonTimestampPrefix);

    // Write the type if it is not the same as before.
    if ((flag & FLAG_SAME_TYPE) == 0) {
      out.write(cell.getTypeByte());
    }

    // Write the value if it is not the same as before.
    if ((flag & FLAG_SAME_VALUE) == 0) {
      PrivateCellUtil.writeValue(out, cell, vLength);
    }
  }
  return kLength + vLength + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE;
}
 
Example 19
Source File: BufferedDataBlockEncoder.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public int seekToKeyInBlock(Cell seekCell, boolean seekBefore) {
  int rowCommonPrefix = 0;
  int familyCommonPrefix = 0;
  int qualCommonPrefix = 0;
  previous.invalidate();
  do {
    int comp;
    keyOnlyKV.setKey(current.keyBuffer, 0, current.keyLength);
    if (current.lastCommonPrefix != 0) {
      // The KV format has row key length also in the byte array. The
      // common prefix
      // includes it. So we need to subtract to find out the common prefix
      // in the
      // row part alone
      rowCommonPrefix = Math.min(rowCommonPrefix, current.lastCommonPrefix - 2);
    }
    if (current.lastCommonPrefix <= 2) {
      rowCommonPrefix = 0;
    }
    rowCommonPrefix += findCommonPrefixInRowPart(seekCell, keyOnlyKV, rowCommonPrefix);
    comp = compareCommonRowPrefix(seekCell, keyOnlyKV, rowCommonPrefix);
    if (comp == 0) {
      comp = compareTypeBytes(seekCell, keyOnlyKV);
      if (comp == 0) {
        // Subtract the fixed row key length and the family key fixed length
        familyCommonPrefix = Math.max(
            0,
            Math.min(familyCommonPrefix,
                current.lastCommonPrefix - (3 + keyOnlyKV.getRowLength())));
        familyCommonPrefix += findCommonPrefixInFamilyPart(seekCell, keyOnlyKV,
            familyCommonPrefix);
        comp = compareCommonFamilyPrefix(seekCell, keyOnlyKV, familyCommonPrefix);
        if (comp == 0) {
          // subtract the rowkey fixed length and the family key fixed
          // length
          qualCommonPrefix = Math.max(
              0,
              Math.min(
                  qualCommonPrefix,
                  current.lastCommonPrefix
                      - (3 + keyOnlyKV.getRowLength() + keyOnlyKV.getFamilyLength())));
          qualCommonPrefix += findCommonPrefixInQualifierPart(seekCell, keyOnlyKV,
              qualCommonPrefix);
          comp = compareCommonQualifierPrefix(seekCell, keyOnlyKV, qualCommonPrefix);
          if (comp == 0) {
            comp = CellComparator.getInstance().compareTimestamps(seekCell, keyOnlyKV);
            if (comp == 0) {
              // Compare types. Let the delete types sort ahead of puts;
              // i.e. types
              // of higher numbers sort before those of lesser numbers.
              // Maximum
              // (255)
              // appears ahead of everything, and minimum (0) appears
              // after
              // everything.
              comp = (0xff & keyOnlyKV.getTypeByte()) - (0xff & seekCell.getTypeByte());
            }
          }
        }
      }
    }
    if (comp == 0) { // exact match
      if (seekBefore) {
        if (!previous.isValid()) {
          // The caller (seekBefore) has to ensure that we are not at the
          // first key in the block.
          throw new IllegalStateException("Cannot seekBefore if "
              + "positioned at the first key in the block: key="
              + Bytes.toStringBinary(seekCell.getRowArray()));
        }
        moveToPrevious();
        return 1;
      }
      return 0;
    }

    if (comp < 0) { // already too large, check previous
      if (previous.isValid()) {
        moveToPrevious();
      } else {
        return HConstants.INDEX_KEY_MAGIC; // using optimized index key
      }
      return 1;
    }

    // move to next, if more data is available
    if (currentBuffer.hasRemaining()) {
      previous.copyFromNext(current);
      decodeNext();
      current.setKey(current.keyBuffer, current.memstoreTS);
    } else {
      break;
    }
  } while (true);

  // we hit the end of the block, not an exact match
  return 1;
}
 
Example 20
Source File: Batch.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public void add(Cell kv){
  if (pointDeleteCode != kv.getTypeByte()) {
    allPointDeletes = false;
  }
  batch.add(kv);
}