Java Code Examples for org.apache.hadoop.hbase.KeyValue#getBuffer()

public ReturnCode resolveColumn(KeyValue value) {
    // Always set key, in case we never find a key value column of interest,
    // and our expression uses row key columns.
    setKey(value);
    byte[] buf = value.getBuffer();
    Object ptr = setColumnKey(buf, value.getFamilyOffset(), value.getFamilyLength(), buf, value.getQualifierOffset(), value.getQualifierLength());
    KeyValueRef ref = foundColumns.get(ptr);
    if (ref == null) {
        // Return INCLUDE here. Although this filter doesn't need this KV
        // it should still be projected into the Result
        return ReturnCode.INCLUDE;
    }
    // Since we only look at the latest key value for a given column,
    // we are not interested in older versions
    // TODO: test with older versions to confirm this doesn't get tripped
    // This shouldn't be necessary, because a scan only looks at the latest
    // version
    if (ref.keyValue != null) {
        // Can't do NEXT_ROW, because then we don't match the other columns
        // SKIP, INCLUDE, and NEXT_COL seem to all act the same
        return ReturnCode.NEXT_COL;
    }
    ref.keyValue = value;
    refCount++;
    return null;
}
 

private static KeyValue upgradeTo3(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    int newLength = keyValue.getRowLength() + 1;
    byte[] newKey = new byte[newLength];
    newKey[0] = QueryConstants.SEPARATOR_BYTE;
    System.arraycopy(buf, keyValue.getRowOffset(), newKey, 1, keyValue.getRowLength());
    byte[] valueBuf = updateValueIfNecessary(keyValue);
    int valueOffset = keyValue.getValueOffset();
    int valueLength = keyValue.getValueLength();
    if (valueBuf != buf) {
        valueOffset = 0;
        valueLength = valueBuf.length;
    }
    return new KeyValue(newKey, 0, newLength,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), Type.codeToType(keyValue.getType()),
            valueBuf, valueOffset, valueLength);
}
 

private static void logMutationSize(HTableInterface htable, List<Mutation> mutations) {
    long byteSize = 0;
    int keyValueCount = 0;
    for (Mutation mutation : mutations) {
        if (mutation.getFamilyMap() != null) { // Not a Delete of the row
            for (Entry<byte[], List<KeyValue>> entry : mutation.getFamilyMap().entrySet()) {
                if (entry.getValue() != null) {
                    for (KeyValue kv : entry.getValue()) {
                        byteSize += kv.getBuffer().length;
                        keyValueCount++;
                    }
                }
            }
        }
    }
    logger.debug("Sending " + mutations.size() + " mutations for " + Bytes.toString(htable.getTableName()) + " with " + keyValueCount + " key values of total size " + byteSize + " bytes");
}
 

public static ValueGetter createGetterFromKeyValues(Collection<KeyValue> pendingUpdates) {
    final Map<ReferencingColumn, ImmutableBytesPtr> valueMap = Maps.newHashMapWithExpectedSize(pendingUpdates
            .size());
    for (KeyValue kv : pendingUpdates) {
        // create new pointers to each part of the kv
        ImmutableBytesPtr family = new ImmutableBytesPtr(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength());
        ImmutableBytesPtr qual = new ImmutableBytesPtr(kv.getBuffer(), kv.getQualifierOffset(),
                kv.getQualifierLength());
        ImmutableBytesPtr value = new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
        valueMap.put(new ReferencingColumn(family, qual), value);
    }
    return new ValueGetter() {
        @Override
        public ImmutableBytesPtr getLatestValue(ColumnReference ref) throws IOException {
            return valueMap.get(ReferencingColumn.wrap(ref));
        }
    };
}
 

@Override
public Cell getNextCellHint(Cell currentKV) {
  // this might be a little excessive right now - better safe than sorry though, so we don't mess
  // with other filters too much.
  KeyValue kv = null;
  try {
      kv = KeyValueUtil.ensureKeyValue(currentKV).clone();
  } catch (CloneNotSupportedException e) {
      // the exception should not happen at all
      throw new IllegalArgumentException(e);
  }
  int offset =kv.getTimestampOffset();
  //set the timestamp in the buffer
  @SuppressWarnings("deprecation")
  byte[] buffer = kv.getBuffer();
  byte[] ts = Bytes.toBytes(this.ts);
  System.arraycopy(ts, 0, buffer, offset, ts.length);

  return kv;
}
 

@Override
public KeyValue getNextKeyHint(KeyValue currentKV) {
  // this might be a little excessive right now - better safe than sorry though, so we don't mess
  // with other filters too much.
  KeyValue kv = currentKV.deepCopy();
  int offset =kv.getTimestampOffset();
  //set the timestamp in the buffer
  byte[] buffer = kv.getBuffer();
  byte[] ts = Bytes.toBytes(this.ts);
  System.arraycopy(ts, 0, buffer, offset, ts.length);

  return kv;
}
 

@Override
public ReturnCode filterKeyValue(KeyValue keyValue) {
    if (this.matchedColumn) {
      // We already found and matched the single column, all keys now pass
      // TODO: why won't this cause earlier versions of a kv to be included?
      return ReturnCode.INCLUDE;
    }
    if (this.foundColumn()) {
      // We found all the columns, but did not match the expression, so skip to next row
      return ReturnCode.NEXT_ROW;
    }
    byte[] buf = keyValue.getBuffer();
    if (compare(buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(), buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength()) != 0) {
        // Remember the key in case this is the only key value we see.
        // We'll need it if we have row key columns too.
        inputTuple.setKey(keyValue);
        // This is a key value we're not interested in
        // TODO: use NEXT_COL when bug fix comes through that includes the row still
        return ReturnCode.INCLUDE;
    }
    inputTuple.setKeyValue(keyValue);

    // We have the columns, so evaluate here
    if (!Boolean.TRUE.equals(evaluate(inputTuple))) {
        return ReturnCode.NEXT_ROW;
    }
    this.matchedColumn = true;
    return ReturnCode.INCLUDE;
}
 

/**
 * Binary search for latest column value without allocating memory in the process
 * @param kvs
 * @param family
 * @param qualifier
 */
public static KeyValue getColumnLatest(List<KeyValue>kvs, byte[] family, byte[] qualifier) {
    if (kvs.size() == 0) {
    	return null;
    }
    KeyValue row = kvs.get(0);
    Comparator<KeyValue> comp = new SearchComparator(row.getBuffer(), row.getRowOffset(), row.getRowLength(), family, qualifier);
    // pos === ( -(insertion point) - 1)
    int pos = Collections.binarySearch(kvs, null, comp);
    // never will exact match
    if (pos < 0) {
      pos = (pos+1) * -1;
      // pos is now insertion point
    }
    if (pos == kvs.size()) {
      return null; // doesn't exist
    }

    KeyValue kv = kvs.get(pos);
    if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(),
            family, 0, family.length) != 0) {
        return null;
    }
    if (Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(),
            qualifier, 0, qualifier.length) != 0) {
        return null;
    }
    return kv;
}
 

private static byte[] updateValueIfNecessary(KeyValue keyValue) {
    byte[] buf = keyValue.getBuffer();
    if (Bytes.compareTo(
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(), 
            PhoenixDatabaseMetaData.DATA_TYPE_BYTES, 0, PhoenixDatabaseMetaData.DATA_TYPE_BYTES.length) == 0) {
        int sqlType = PDataType.INTEGER.getCodec().decodeInt(buf, keyValue.getValueOffset(), null);
        // Switch the DATA_TYPE index for TIME types, as they currently don't support negative values
        // In 3.0, we'll switch them to our serialized LONG type instead, as that way we can easily
        // support negative types going forward.
        if (sqlType >= 0 && sqlType < OLD_TO_NEW_DATA_TYPE_3_0.length && OLD_TO_NEW_DATA_TYPE_3_0[sqlType] != null) {
            return OLD_TO_NEW_DATA_TYPE_3_0[sqlType];
        }
    }
    return buf;
}
 

/**
 * @param ref
 * @return the first value on the scanner for the given column
 */
private ImmutableBytesPtr get(ColumnReference ref) throws IOException {
  KeyValue first = ref.getFirstKeyValueForRow(row);
  if (!scan.seek(first)) {
    return null;
  }
  // there is a next value - we only care about the current value, so we can just snag that
  KeyValue next = scan.next();
  if (ref.matches(next)) {
    return new ImmutableBytesPtr(next.getBuffer(), next.getValueOffset(), next.getValueLength());
  }
  return null;
}
 

private Cell createOffheapKVWithTags(int noOfTags) {
  List<Tag> tags = new ArrayList<>();
  for (int i = 0; i < noOfTags; i++) {
    tags.add(new ArrayBackedTag((byte) i, "tagValue" + i));
  }
  KeyValue kv = new KeyValue(ROW, CF, Q, 1234L, V, tags);
  ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length);
  ByteBufferUtils.copyFromArrayToBuffer(dbb, kv.getBuffer(), 0, kv.getBuffer().length);
  ByteBufferKeyValue offheapKV = new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length, 0);
  return offheapKV;
}
 

private void regexPatternFilterTests(Filter filter)
    throws Exception {
  KeyValue cell = new KeyValue(ROW, COLUMN_FAMILY, COLUMN_QUALIFIER,
    FULLSTRING_1);
  assertTrue("regexTrue",
    filter.filterCell(cell) == Filter.ReturnCode.INCLUDE);
  byte[] buffer = cell.getBuffer();
  Cell c = new ByteBufferKeyValue(ByteBuffer.wrap(buffer), 0, buffer.length);
  assertTrue("regexTrue", filter.filterCell(c) == Filter.ReturnCode.INCLUDE);
  assertFalse("regexFilterAllRemaining", filter.filterAllRemaining());
  assertFalse("regexFilterNotNull", filter.filterRow());
}
 

private ByteBufferKeyValue createOffheapKV(int noOfTags) {
  byte[] row = Bytes.toBytes("myRow");
  byte[] cf = Bytes.toBytes("myCF");
  byte[] q = Bytes.toBytes("myQualifier");
  byte[] value = Bytes.toBytes("myValue");
  List<Tag> tags = new ArrayList<>(noOfTags);
  for (int i = 1; i <= noOfTags; i++) {
    tags.add(new ArrayBackedTag((byte) i, Bytes.toBytes("tagValue" + i)));
  }
  KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags);
  ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length);
  dbb.put(kv.getBuffer());
  return new ByteBufferKeyValue(dbb, 0, kv.getBuffer().length);
}
 

@SuppressWarnings("deprecation")
private static KeyValue addSaltByte(KeyValue keyValue, int nSaltBuckets) {
    byte[] buf = keyValue.getBuffer();
    int length = keyValue.getRowLength();
    int offset = keyValue.getRowOffset();
    boolean isViewSeq = length > SEQ_PREFIX_BYTES.length && Bytes.compareTo(SEQ_PREFIX_BYTES, 0, SEQ_PREFIX_BYTES.length, buf, offset, SEQ_PREFIX_BYTES.length) == 0;
    if (!isViewSeq && nSaltBuckets == 0) {
        return null;
    }
    byte[] newBuf;
    if (isViewSeq) { // We messed up the name for the sequences for view indexes so we'll take this opportunity to fix it
        if (buf[length-1] == 0) { // Global indexes on views have trailing null byte
            length--;
        }
        byte[][] rowKeyMetaData = new byte[3][];
        SchemaUtil.getVarChars(buf, offset, length, 0, rowKeyMetaData);
        byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
        byte[] unprefixedSchemaName = new byte[schemaName.length - MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length];
        System.arraycopy(schemaName, MetaDataUtil.VIEW_INDEX_SEQUENCE_PREFIX_BYTES.length, unprefixedSchemaName, 0, unprefixedSchemaName.length);
        byte[] tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
        PName physicalName = PNameFactory.newName(unprefixedSchemaName);
        // Reformulate key based on correct data
        newBuf = MetaDataUtil.getViewIndexSequenceKey(tableName == null ? null : Bytes.toString(tableName), physicalName, nSaltBuckets).getKey();
    } else {
        newBuf = new byte[length + 1];
        System.arraycopy(buf, offset, newBuf, SaltingUtil.NUM_SALTING_BYTES, length);
        newBuf[0] = SaltingUtil.getSaltingByte(newBuf, SaltingUtil.NUM_SALTING_BYTES, length, nSaltBuckets);
    }
    return new KeyValue(newBuf, 0, newBuf.length,
            buf, keyValue.getFamilyOffset(), keyValue.getFamilyLength(),
            buf, keyValue.getQualifierOffset(), keyValue.getQualifierLength(),
            keyValue.getTimestamp(), KeyValue.Type.codeToType(keyValue.getType()),
            buf, keyValue.getValueOffset(), keyValue.getValueLength());
}
 

static byte[] getRawBytes(Result r) {
    KeyValue firstKV = PhoenixKeyValueUtil.maybeCopyCell(r.rawCells()[0]);
    return firstKV.getBuffer();
}
 

public ProjectedValueTuple projectResults(Tuple tuple) {
	byte[] bytesValue = schema.toBytes(tuple, expressions, valueSet, ptr);
	KeyValue base = tuple.getValue(0);
    return new ProjectedValueTuple(base.getBuffer(), base.getRowOffset(), base.getRowLength(), base.getTimestamp(), bytesValue, valueSet.getEstimatedLength());
}
 

/**
 * Verify if all data block encoders are working properly.
 *
 * @param scanner Of file which was compressed.
 * @param kvLimit Maximal count of KeyValue which will be processed.
 * @return true if all data block encoders compressed/decompressed correctly.
 * @throws IOException thrown if scanner is invalid
 */
public boolean verifyCodecs(final KeyValueScanner scanner, final int kvLimit)
    throws IOException {
  KeyValue currentKv;

  scanner.seek(KeyValue.LOWESTKEY);
  List<Iterator<Cell>> codecIterators = new ArrayList<>();
  for(EncodedDataBlock codec : codecs) {
    codecIterators.add(codec.getIterator(HFileBlock.headerSize(useHBaseChecksum)));
  }

  int j = 0;
  while ((currentKv = KeyValueUtil.ensureKeyValue(scanner.next())) != null && j < kvLimit) {
    // Iterates through key/value pairs
    ++j;
    for (Iterator<Cell> it : codecIterators) {
      Cell c = it.next();
      KeyValue codecKv = KeyValueUtil.ensureKeyValue(c);
      if (codecKv == null || 0 != Bytes.compareTo(
          codecKv.getBuffer(), codecKv.getOffset(), codecKv.getLength(),
          currentKv.getBuffer(), currentKv.getOffset(),
          currentKv.getLength())) {
        if (codecKv == null) {
          LOG.error("There is a bug in codec " + it +
              " it returned null KeyValue,");
        } else {
          int prefix = 0;
          int limitLength = 2 * Bytes.SIZEOF_INT +
              Math.min(codecKv.getLength(), currentKv.getLength());
          while (prefix < limitLength &&
              codecKv.getBuffer()[prefix + codecKv.getOffset()] ==
              currentKv.getBuffer()[prefix + currentKv.getOffset()]) {
            prefix++;
          }

          LOG.error("There is bug in codec " + it.toString() +
              "\n on element " + j +
              "\n codecKv.getKeyLength() " + codecKv.getKeyLength() +
              "\n codecKv.getValueLength() " + codecKv.getValueLength() +
              "\n codecKv.getLength() " + codecKv.getLength() +
              "\n currentKv.getKeyLength() " + currentKv.getKeyLength() +
              "\n currentKv.getValueLength() " + currentKv.getValueLength() +
              "\n codecKv.getLength() " + currentKv.getLength() +
              "\n currentKV rowLength " + currentKv.getRowLength() +
              " familyName " + currentKv.getFamilyLength() +
              " qualifier " + currentKv.getQualifierLength() +
              "\n prefix " + prefix +
              "\n codecKv   '" + Bytes.toStringBinary(codecKv.getBuffer(),
                  codecKv.getOffset(), prefix) + "' diff '" +
                  Bytes.toStringBinary(codecKv.getBuffer(),
                      codecKv.getOffset() + prefix, codecKv.getLength() -
                      prefix) + "'" +
              "\n currentKv '" + Bytes.toStringBinary(
                 currentKv.getBuffer(),
                 currentKv.getOffset(), prefix) + "' diff '" +
                 Bytes.toStringBinary(currentKv.getBuffer(),
                     currentKv.getOffset() + prefix, currentKv.getLength() -
                     prefix) + "'"
              );
        }
        return false;
      }
    }
  }

  LOG.info("Verification was successful!");

  return true;
}
 

public static List<Mutation> generateIndexData(final PTable table, PTable index,
        List<Mutation> dataMutations, ImmutableBytesWritable ptr, KeyValueBuilder builder)
        throws SQLException {
    try {
        IndexMaintainer maintainer = index.getIndexMaintainer(table);
        maintainer.setKvBuilder(builder);
        List<Mutation> indexMutations = Lists.newArrayListWithExpectedSize(dataMutations.size());
       for (final Mutation dataMutation : dataMutations) {
            long ts = MetaDataUtil.getClientTimeStamp(dataMutation);
            ptr.set(dataMutation.getRow());
            if (dataMutation instanceof Put) {
                // TODO: is this more efficient than looking in our mutation map
                // using the key plus finding the PColumn?
                ValueGetter valueGetter = new ValueGetter() {
    
                    @Override
                    public ImmutableBytesPtr getLatestValue(ColumnReference ref) {
                        // Always return null for our empty key value, as this will cause the index
                        // maintainer to always treat this Put as a new row.
                        if (isEmptyKeyValue(table, ref)) {
                            return null;
                        }
                        Map<byte [], List<KeyValue>> familyMap = dataMutation.getFamilyMap();
                        byte[] family = ref.getFamily();
                        List<KeyValue> kvs = familyMap.get(family);
                        if (kvs == null) {
                            return null;
                        }
                        byte[] qualifier = ref.getQualifier();
                        for (KeyValue kv : kvs) {
                            if (Bytes.compareTo(kv.getBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), family, 0, family.length) == 0 &&
                                Bytes.compareTo(kv.getBuffer(), kv.getQualifierOffset(), kv.getQualifierLength(), qualifier, 0, qualifier.length) == 0) {
                                return new ImmutableBytesPtr(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength());
                            }
                        }
                        return null;
                    }
                    
                };
                indexMutations.add(maintainer.buildUpdateMutation(valueGetter, ptr, ts));
            } else {
                if (!maintainer.getIndexedColumns().isEmpty()) {
                    throw new SQLExceptionInfo.Builder(SQLExceptionCode.NO_DELETE_IF_IMMUTABLE_INDEX).setSchemaName(table.getSchemaName().getString())
                    .setTableName(table.getTableName().getString()).build().buildException();
                }
                indexMutations.add(maintainer.buildDeleteMutation(ptr, ts));
            }
        }
        return indexMutations;
    } catch (IOException e) {
        throw new SQLException(e);
    }
}
 

@SuppressWarnings("deprecation")
static byte[] getRawBytes(Result r) {
    KeyValue firstKV = org.apache.hadoop.hbase.KeyValueUtil.ensureKeyValue(r.rawCells()[0]);
    return firstKV.getBuffer();
}