Java Code Examples for org.apache.hadoop.hbase.KeyValue#getBuffer()

The following examples show how to use org.apache.hadoop.hbase.KeyValue#getBuffer() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix   File: MaxTimestampFilter.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Cell getNextCellHint(Cell currentKV) {
  // this might be a little excessive right now - better safe than sorry though, so we don't mess
  // with other filters too much.
  KeyValue kv = null;
  try {
      kv = KeyValueUtil.ensureKeyValue(currentKV).clone();
  } catch (CloneNotSupportedException e) {
      // the exception should not happen at all
      throw new IllegalArgumentException(e);
  }
  int offset =kv.getTimestampOffset();
  //set the timestamp in the buffer
  @SuppressWarnings("deprecation")
  byte[] buffer = kv.getBuffer();
  byte[] ts = Bytes.toBytes(this.ts);
  System.arraycopy(ts, 0, buffer, offset, ts.length);

  return kv;
}
 
Example 2
public static ValueGetter createGetterFromKeyValues(Collection<KeyValue> pendingUpdates) {
    final Map<ReferencingColumn, ImmutableBytesPtr> valueMap = Maps.newHashMapWithExpectedSize(pendingUpdates
            .size());
    for (KeyValue kv : pendingUpdates) {
        // create new pointers to each part of the kv
        ImmutableBytesPtr family = new ImmutableBytesPtr(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength());
        ImmutableBytesPtr qual = new ImmutableBytesPtr(kv.getBuffer(), kv.getQualifierOffset(),
                kv.getQualifierLength());
        ImmutableBytesPtr value = new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
        valueMap.put(new ReferencingColumn(family, qual), value);
    }
    return new ValueGetter() {
        @Override
        public ImmutableBytesPtr getLatestValue(ColumnReference ref) throws IOException {
            return valueMap.get(ReferencingColumn.wrap(ref));
        }
    };
}
 
Example 3
private static void logMutationSize(HTableInterface htable, List<Mutation> mutations) {
    long byteSize = 0;
    int keyValueCount = 0;
    for (Mutation mutation : mutations) {
        if (mutation.getFamilyMap() != null) { // Not a Delete of the row
            for (Entry<byte[], List<KeyValue>> entry : mutation.getFamilyMap().entrySet()) {
                if (entry.getValue() != null) {
                    for (KeyValue kv : entry.getValue()) {
                        byteSize += kv.getBuffer().length;
                        keyValueCount++;
                    }
                }
            }
        }
    }
    logger.debug("Sending " + mutations.size() + " mutations for " + Bytes.toString(htable.getTableName()) + " with " + keyValueCount + " key values of total size " + byteSize + " bytes");
}
 
Example 4
private static KeyValue upgradeTo3(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    int newLength = keyValue.getRowLength() + 1;
    byte[] newKey = new byte[newLength];
    newKey[0] = QueryConstants.SEPARATOR_BYTE;
    System.arraycopy(buf, keyValue.getRowOffset(), newKey, 1, keyValue.getRowLength());
    byte[] valueBuf = updateValueIfNecessary(keyValue);
    int valueOffset = keyValue.getValueOffset();
    int valueLength = keyValue.getValueLength();
    if (valueBuf != buf) {
        valueOffset = 0;
        valueLength = valueBuf.length;
    }
    return new KeyValue(newKey, 0, newLength,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), Type.codeToType(keyValue.getType()),
            valueBuf, valueOffset, valueLength);
}
 
Example 5
public ReturnCode resolveColumn(KeyValue value) {
    // Always set key, in case we never find a key value column of interest,
    // and our expression uses row key columns.
    setKey(value);
    byte[] buf = value.getBuffer();
    Object ptr = setColumnKey(buf, value.getFamilyOffset(), value.getFamilyLength(), buf, value.getQualifierOffset(), value.getQualifierLength());
    KeyValueRef ref = foundColumns.get(ptr);
    if (ref == null) {
        // Return INCLUDE here. Although this filter doesn't need this KV
        // it should still be projected into the Result
        return ReturnCode.INCLUDE;
    }
    // Since we only look at the latest key value for a given column,
    // we are not interested in older versions
    // TODO: test with older versions to confirm this doesn't get tripped
    // This shouldn't be necessary, because a scan only looks at the latest
    // version
    if (ref.keyValue != null) {
        // Can't do NEXT_ROW, because then we don't match the other columns
        // SKIP, INCLUDE, and NEXT_COL seem to all act the same
        return ReturnCode.NEXT_COL;
    }
    ref.keyValue = value;
    refCount++;
    return null;
}
 
Example 6
Source Project: phoenix   File: UpgradeUtil.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private static KeyValue addSaltByte(KeyValue keyValue, int nSaltBuckets) {
    byte[] buf = keyValue.getBuffer();
    int length = keyValue.getRowLength();
    int offset = keyValue.getRowOffset();
    boolean isViewSeq = length > SEQ_PREFIX_BYTES.length && Bytes.compareTo(SEQ_PREFIX_BYTES, 0, SEQ_PREFIX_BYTES.length, buf, offset, SEQ_PREFIX_BYTES.length) == 0;
    if (!isViewSeq && nSaltBuckets == 0) {
        return null;
    }
    byte[] newBuf;
    if (isViewSeq) { // We messed up the name for the sequences for view indexes so we'll take this opportunity to fix it
        if (buf[length-1] == 0) { // Global indexes on views have trailing null byte
            length--;
        }
        byte[][] rowKeyMetaData = new byte[3][];
        SchemaUtil.getVarChars(buf, offset, length, 0, rowKeyMetaData);
        byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
        byte[] unprefixedSchemaName = new byte[schemaName.length - MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length];
        System.arraycopy(schemaName, MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length, unprefixedSchemaName, 0, unprefixedSchemaName.length);
        byte[] tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
        PName physicalName = PNameFactory.newName(unprefixedSchemaName);
        // Reformulate key based on correct data
        newBuf = MetaDataUtil.getViewIndexSequenceKey(tableName == null ? null : Bytes.toString(tableName), physicalName, nSaltBuckets).getKey();
    } else {
        newBuf = new byte[length + 1];
        System.arraycopy(buf, offset, newBuf, SaltingUtil.NUM_SALTING_BYTES, length);
        newBuf[0] = SaltingUtil.getSaltingByte(newBuf, SaltingUtil.NUM_SALTING_BYTES, length, nSaltBuckets);
    }
    return new KeyValue(newBuf, 0, newBuf.length,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), KeyValue.Type.codeToType(keyValue.getType()),
            buf, keyValue.getValueOffset(), keyValue.getValueLength());
}
 
Example 7
private ByteBufferKeyValue createOffheapKV(int noOfTags) {
  byte[] row = Bytes.toBytes("myRow");
  byte[] cf = Bytes.toBytes("myCF");
  byte[] q = Bytes.toBytes("myQualifier");
  byte[] value = Bytes.toBytes("myValue");
  List<Tag> tags = new ArrayList<>(noOfTags);
  for (int i = 1; i <= noOfTags; i++) {
    tags.add(new ArrayBackedTag((byte) i, Bytes.toBytes("tagValue" + i)));
  }
  KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags);
  ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length);
  dbb.put(kv.getBuffer());
  return new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length);
}
 
Example 8
private void regexPatternFilterTests(Filter filter)
    throws Exception {
  KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER,
    FULLSTRING_1);
  assertTrue("regexTrue",
    filter.filterCell(cell) == Filter.ReturnCode.INCLUDE);
  byte[] buffer = cell.getBuffer();
  Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length);
  assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE);
  assertFalse("regexFilterAllRemaining", filter.filterAllRemaining());
  assertFalse("regexFilterNotNull", filter.filterRow());
}
 
Example 9
Source Project: hbase   File: TestTagCompressionContext.java    License: Apache License 2.0 5 votes vote down vote up
private Cell createOffheapKVWithTags(int noOfTags) {
  List<Tag> tags = new ArrayList<>();
  for (int i = 0; i < noOfTags; i++) {
    tags.add(new ArrayBackedTag((byte) i, "tagValue" + i));
  }
  KeyValue kv = new KeyValue(ROW, CF, Q, 1234L, V, tags);
  ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length);
  ByteBufferUtils.copyFromArrayToBuffer(dbb, kv.getBuffer(), 0, kv.getBuffer().length);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length, 0);
  return offheapKV;
}
 
Example 10
/**
 * @param ref
 * @return the first value on the scanner for the given column
 */
private ImmutableBytesPtr get(ColumnReference ref) throws IOException {
  KeyValue first = ref.getFirstKeyValueForRow(row);
  if (!scan.seek(first)) {
    return null;
  }
  // there is a next value - we only care about the current value, so we can just snag that
  KeyValue next = scan.next();
  if (ref.matches(next)) {
    return new ImmutableBytesPtr(next.getBuffer(), next.getValueOffset(), next.getValueLength());
  }
  return null;
}
 
Example 11
@Override
public KeyValue getNextKeyHint(KeyValue currentKV) {
  // this might be a little excessive right now - better safe than sorry though, so we don't mess
  // with other filters too much.
  KeyValue kv = currentKV.deepCopy();
  int offset =kv.getTimestampOffset();
  //set the timestamp in the buffer
  byte[] buffer = kv.getBuffer();
  byte[] ts = Bytes.toBytes(this.ts);
  System.arraycopy(ts, 0, buffer, offset, ts.length);

  return kv;
}
 
Example 12
private static byte[] updateValueIfNecessary(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    if (Bytes.compareTo(
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(), 
            PhoenixDatabaseMetaData.DATA_TYPE_BYTES, 0, PhoenixDatabaseMetaData.DATA_TYPE_BYTES.length) == 0) {
        int sqlType = PDataType.INTEGER.getCodec().decodeInt(buf, keyValue.getValueOffset(), null);
        // Switch the DATA_TYPE index for TIME types, as they currently don't support negative values
        // In 3.0, we'll switch them to our serialized LONG type instead, as that way we can easily
        // support negative types going forward.
        if (sqlType >= 0 && sqlType < OLD_TO_NEW_DATA_TYPE_3_0.length && OLD_TO_NEW_DATA_TYPE_3_0[sqlType] != null) {
            return OLD_TO_NEW_DATA_TYPE_3_0[sqlType];
        }
    }
    return buf;
}
 
Example 13
/**
 * Binary search for latest column value without allocating memory in the process
 * @param kvs
 * @param family
 * @param qualifier
 */
public static KeyValue getColumnLatest(List<KeyValue>kvs, byte[] family, byte[] qualifier) {
    if (kvs.size() == 0) {
    	return null;
    }
    KeyValue row = kvs.get(0);
    Comparator<KeyValue> comp = new SearchComparator(row.getBuffer(), row.getRowOffset(), row.getRowLength(), family, qualifier);
    // pos === ( -(insertion point) - 1)
    int pos = Collections.binarySearch(kvs, null, comp);
    // never will exact match
    if (pos < 0) {
      pos = (pos+1) * -1;
      // pos is now insertion point
    }
    if (pos == kvs.size()) {
      return null; // doesn't exist
    }

    KeyValue kv = kvs.get(pos);
    if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
            family, 0, family.length) != 0) {
        return null;
    }
    if (Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(),
            qualifier, 0, qualifier.length) != 0) {
        return null;
    }
    return kv;
}
 
Example 14
@Override
public ReturnCode filterKeyValue(KeyValue keyValue) {
    if (this.matchedColumn) {
      // We already found and matched the single column, all keys now pass
      // TODO: why won't this cause earlier versions of a kv to be included?
      return ReturnCode.INCLUDE;
    }
    if (this.foundColumn()) {
      // We found all the columns, but did not match the expression, so skip to next row
      return ReturnCode.NEXT_ROW;
    }
    byte[] buf = keyValue.getBuffer();
    if (compare(buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(), buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength()) != 0) {
        // Remember the key in case this is the only key value we see.
        // We'll need it if we have row key columns too.
        inputTuple.setKey(keyValue);
        // This is a key value we're not interested in
        // TODO: use NEXT_COL when bug fix comes through that includes the row still
        return ReturnCode.INCLUDE;
    }
    inputTuple.setKeyValue(keyValue);

    // We have the columns, so evaluate here
    if (!Boolean.TRUE.equals(evaluate(inputTuple))) {
        return ReturnCode.NEXT_ROW;
    }
    this.matchedColumn = true;
    return ReturnCode.INCLUDE;
}
 
Example 15
Source Project: phoenix   File: ResultUtil.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("deprecation")
static byte[] getRawBytes(Result r) {
    KeyValue firstKV = org.apache.hadoop.hbase.KeyValueUtil.ensureKeyValue(r.rawCells()[0]);
    return firstKV.getBuffer();
}
 
Example 16
Source Project: hbase   File: DataBlockEncodingTool.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Verify if all data block encoders are working properly.
 *
 * @param scanner Of file which was compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @return true if all data block encoders compressed/decompressed correctly.
 * @throws IOException thrown if scanner is invalid
 */
public boolean verifyCodecs(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  KeyValue currentKv;

  scanner.seek(KeyValue.LOWESTKEY);
  List<Iterator<Cell>> codecIterators = new ArrayList<>();
  for(EncodedDataBlock codec : codecs) {
    codecIterators.add(codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)));
  }

  int j = 0;
  while ((currentKv = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    ++j;
    for (Iterator<Cell> it : codecIterators) {
      Cell c = it.next();
      KeyValue codecKv = KeyValueUtil.ensureKeyValue(c);
      if (codecKv == null || 0 != Bytes.compareTo(
          codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(),
          currentKv.getBuffer(), currentKv.getOffset(),
          currentKv.getLength())) {
        if (codecKv == null) {
          LOG.error("There is a bug in codec " + it +
              " it returned null KeyValue,");
        } else {
          int prefix = 0;
          int limitLength = 2 * Bytes.SIZEOF_INT +
              Math.min(codecKv.getLength(), currentKv.getLength());
          while (prefix < limitLength &&
              codecKv.getBuffer()[prefix + codecKv.getOffset()] ==
              currentKv.getBuffer()[prefix + currentKv.getOffset()]) {
            prefix++;
          }

          LOG.error("There is bug in codec " + it.toString() +
              "\n on element " + j +
              "\n codecKv.getKeyLength() " + codecKv.getKeyLength() +
              "\n codecKv.getValueLength() " + codecKv.getValueLength() +
              "\n codecKv.getLength() " + codecKv.getLength() +
              "\n currentKv.getKeyLength() " + currentKv.getKeyLength() +
              "\n currentKv.getValueLength() " + currentKv.getValueLength() +
              "\n codecKv.getLength() " + currentKv.getLength() +
              "\n currentKV rowLength " + currentKv.getRowLength() +
              " familyName " + currentKv.getFamilyLength() +
              " qualifier " + currentKv.getQualifierLength() +
              "\n prefix " + prefix +
              "\n codecKv   '" + Bytes.toStringBinary(codecKv.getBuffer(),
                  codecKv.getOffset(), prefix) + "' diff '" +
                  Bytes.toStringBinary(codecKv.getBuffer(),
                      codecKv.getOffset() + prefix, codecKv.getLength() -
                      prefix) + "'" +
              "\n currentKv '" + Bytes.toStringBinary(
                 currentKv.getBuffer(),
                 currentKv.getOffset(), prefix) + "' diff '" +
                 Bytes.toStringBinary(currentKv.getBuffer(),
                     currentKv.getOffset() + prefix, currentKv.getLength() -
                     prefix) + "'"
              );
        }
        return false;
      }
    }
  }

  LOG.info("Verification was successful!");

  return true;
}
 
Example 17
Source Project: phoenix   File: ResultUtil.java    License: Apache License 2.0 4 votes vote down vote up
static byte[] getRawBytes(Result r) {
    KeyValue firstKV = PhoenixKeyValueUtil.maybeCopyCell(r.rawCells()[0]);
    return firstKV.getBuffer();
}
 
Example 18
public ProjectedValueTuple projectResults(Tuple tuple) {
	byte[] bytesValue = schema.toBytes(tuple, expressions, valueSet, ptr);
	KeyValue base = tuple.getValue(0);
    return new ProjectedValueTuple(base.getBuffer(), base.getRowOffset(), base.getRowLength(), base.getTimestamp(), bytesValue, valueSet.getEstimatedLength());
}
 
Example 19
public static List<Mutation> generateIndexData(final PTable table, PTable index,
        List<Mutation> dataMutations, ImmutableBytesWritable ptr, KeyValueBuilder builder)
        throws SQLException {
    try {
        IndexMaintainer maintainer = index.getIndexMaintainer(table);
        maintainer.setKvBuilder(builder);
        List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
       for (final Mutation dataMutation : dataMutations) {
            long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
            ptr.set(dataMutation.getRow());
            if (dataMutation instanceof Put) {
                // TODO: is this more efficient than looking in our mutation map
                // using the key plus finding the PColumn?
                ValueGetter valueGetter = new ValueGetter() {
    
                    @Override
                    public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
                        // Always return null for our empty key value, as this will cause the index
                        // maintainer to always treat this Put as a new row.
                        if (isEmptyKeyValue(table, ref)) {
                            return null;
                        }
                        Map<byte [], List<KeyValue>> familyMap = dataMutation.getFamilyMap();
                        byte[] family = ref.getFamily();
                        List<KeyValue> kvs = familyMap.get(family);
                        if (kvs == null) {
                            return null;
                        }
                        byte[] qualifier = ref.getQualifier();
                        for (KeyValue kv : kvs) {
                            if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 &&
                                Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) {
                                return new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
                            }
                        }
                        return null;
                    }
                    
                };
                indexMutations.add(maintainer.buildUpdateMutation(valueGetter, ptr, ts));
            } else {
                if (!maintainer.getIndexedColumns().isEmpty()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_DELETE_IF_IMMUTABLE_INDEX).setSchemaName(table.getSchemaName().getString())
                    .setTableName(table.getTableName().getString()).build().buildException();
                }
                indexMutations.add(maintainer.buildDeleteMutation(ptr, ts));
            }
        }
        return indexMutations;
    } catch (IOException e) {
        throw new SQLException(e);
    }
}