Java Code Examples for org.apache.hadoop.hbase.util.Bytes#putLong()

/**
 * @param put an instance of Put
 * @param timestamp  timestamp to be used as cells version
 * @param commitTimestamp  timestamp to be used as commit timestamp
 * @throws IOException if a remote or network exception occurs.
 */
static public Put markPutAsCommitted(Put put, long timestamp, long commitTimestamp) {
    final Put tsput = new Put(put.getRow(), timestamp);
    propagateAttributes(put, tsput);

    Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
    for (List<Cell> kvl : kvs.values()) {
        for (Cell c : kvl) {
            KeyValue kv = KeyValueUtil.ensureKeyValue(c);
            Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), timestamp);
            try {
                tsput.add(kv);
            } catch (IOException e) {
                // The existing Put has this Cell, so the cloned one
                // will never throw an IOException when it's added.
                throw new RuntimeException(e);
            }
            tsput.addColumn(CellUtil.cloneFamily(kv),
                    CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                    kv.getTimestamp(),
                    Bytes.toBytes(commitTimestamp));
        }
    }

    return tsput;
}
 

/**
 * Serialize the value bit set into a byte array. The byte array
 * is expected to have enough room (use {@link #getEstimatedLength()}
 * to ensure enough room exists.
 * @param b the byte array into which to put the serialized bit set
 * @param offset the offset into the byte array
 * @return the incremented offset
 */
public int toBytes(byte[] b, int offset) {
    if (schema == null) {
        return offset;
    }
    // If the total number of possible values is bigger than 16 bits (the
    // size of a short), then serialize the long array followed by the
    // array length.
    if (isVarLength()) {
        short nLongs = (short)((maxSetBit + BITS_PER_LONG) / BITS_PER_LONG);
        for (int i = 0; i < nLongs; i++) {
            offset = Bytes.putLong(b, offset, bits[i]);
        }
        offset = Bytes.putShort(b, offset, nLongs);
    } else { 
        // Else if the number of values is less than or equal to 16,
        // serialize the bits directly into a short.
        offset = Bytes.putShort(b, offset, (short)bits[0]);            
    }
    return offset;
}
 

/**
 * Serialize the value bit set into a byte array. The byte array
 * is expected to have enough room (use {@link #getEstimatedLength()}
 * to ensure enough room exists.
 * @param b the byte array into which to put the serialized bit set
 * @param offset the offset into the byte array
 * @return the incremented offset
 */
public int toBytes(byte[] b, int offset) {
    if (schema == null) {
        return offset;
    }
    // If the total number of possible values is bigger than 16 bits (the
    // size of a short), then serialize the long array followed by the
    // array length.
    if (isVarLength()) {
        short nLongs = (short)((maxSetBit + BITS_PER_LONG) / BITS_PER_LONG);
        for (int i = 0; i < nLongs; i++) {
            offset = Bytes.putLong(b, offset, bits[i]);
        }
        offset = Bytes.putShort(b, offset, nLongs);
    } else { 
        // Else if the number of values is less than or equal to 16,
        // serialize the bits directly into a short.
        offset = Bytes.putShort(b, offset, (short)bits[0]);            
    }
    return offset;
}
 

/**
 * Serialize the value bit set into a byte array. The byte array
 * is expected to have enough room (use {@link #getEstimatedLength()}
 * to ensure enough room exists.
 * @param b the byte array into which to put the serialized bit set
 * @param offset the offset into the byte array
 * @return the incremented offset
 */
public int toBytes(byte[] b, int offset) {
    if (schema == null) {
        return offset;
    }
    // If the total number of possible values is bigger than 16 bits (the
    // size of a short), then serialize the long array followed by the
    // array length.
    if (isVarLength()) {
        short nLongs = (short)((maxSetBit + BITS_PER_LONG - 1) / BITS_PER_LONG);
        for (int i = 0; i < nLongs; i++) {
            offset = Bytes.putLong(b, offset, bits[i]);
        }
        offset = Bytes.putShort(b, offset, nLongs);
    } else { 
        // Else if the number of values is less than or equal to 16,
        // serialize the bits directly into a short.
        offset = Bytes.putShort(b, offset, (short)bits[0]);            
    }
    return offset;
}
 

private static int encodeDouble(double v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
    long l = Double.doubleToLongBits(v);
    l = (l ^ ((l >> Long.SIZE - 1) | Long.MIN_VALUE)) + 1;
    Bytes.putLong(b, o, l);
    return Bytes.SIZEOF_LONG;
}
 

@Override
public int encodeLong(long v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
    if (v < 0) {
        throw newIllegalDataException();
    }
    Bytes.putLong(b, o, v);
    return Bytes.SIZEOF_LONG;
}
 

private static int encodeUnsignedLong(long v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
    if (v < 0) {
        throw new RuntimeException();
    }
    Bytes.putLong(b, o, v);
    return Bytes.SIZEOF_LONG;
}
 

public static int appendKeyTo(final Cell cell, final byte[] output,
    final int offset) {
  int nextOffset = offset;
  nextOffset = Bytes.putShort(output, nextOffset, cell.getRowLength());
  nextOffset = CellUtil.copyRowTo(cell, output, nextOffset);
  nextOffset = Bytes.putByte(output, nextOffset, cell.getFamilyLength());
  nextOffset = CellUtil.copyFamilyTo(cell, output, nextOffset);
  nextOffset = CellUtil.copyQualifierTo(cell, output, nextOffset);
  nextOffset = Bytes.putLong(output, nextOffset, cell.getTimestamp());
  nextOffset = Bytes.putByte(output, nextOffset, cell.getTypeByte());
  return nextOffset;
}
 

/**
 * Write KeyValue format into a byte array.
 * @param row row key
 * @param roffset row offset
 * @param rlength row length
 * @param family family name
 * @param foffset family offset
 * @param flength family length
 * @param qualifier column qualifier
 * @param qoffset qualifier offset
 * @param qlength qualifier length
 * @param timestamp version timestamp
 * @param type key type
 * @param value column value
 * @param voffset value offset
 * @param vlength value length
 * @return The newly created byte array.
 */
private static byte [] createByteArray(final byte [] row, final int roffset,
    final int rlength, final byte [] family, final int foffset, int flength,
    final byte [] qualifier, final int qoffset, int qlength,
    final long timestamp, final Type type,
    final byte [] value, final int voffset,
    int vlength, byte[] tags, int tagsOffset, int tagsLength) {

  checkParameters(row, rlength, family, flength, qlength, vlength);
  RawCell.checkForTagsLength(tagsLength);
  // Allocate right-sized byte array.
  int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
  byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
    tagsLength)];
  // Write key, value and key row length.
  int pos = 0;
  pos = Bytes.putInt(bytes, pos, keyLength);
  pos = Bytes.putInt(bytes, pos, vlength);
  pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
  pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
  pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
  if(flength != 0) {
    pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
  }
  if(qlength != 0) {
    pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
  }
  pos = Bytes.putLong(bytes, pos, timestamp);
  pos = Bytes.putByte(bytes, pos, type.getCode());
  if (value != null && value.length > 0) {
    pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
  }
  // Add the tags after the value part
  if (tagsLength > 0) {
    pos = Bytes.putAsShort(bytes, pos, tagsLength);
    pos = Bytes.putBytes(bytes, pos, tags, tagsOffset, tagsLength);
  }
  return bytes;
}
 

@Override
public int encodeLong(long v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
  if (v < 0) {
    throw newIllegalDataException();
  }
  Bytes.putLong(b, o, v);
  return Bytes.SIZEOF_LONG;
}
 

@Override
public int encodeDouble(double v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
  long l = Double.doubleToLongBits(v);
  l = (l ^ ((l >> Long.SIZE - 1) | Long.MIN_VALUE)) + 1;
  Bytes.putLong(b, o, l);
  return Bytes.SIZEOF_LONG;
}
 

private int serializeGroup(byte[] buffer, int currentPos, String groupName, Map<String, Long> counterMap) throws JobCounterException {
    int pos = currentPos;
    final CounterGroupKey groupKey = getCounterGroup(groupName);
    if (groupKey == null) {
        throw new JobCounterException("Group name " + groupName + " is not defined");
    }
    Bytes.putInt(buffer, pos, groupKey.getIndex());
    pos += 4;
    int totalCounterNumberPos = pos;
    pos += 4;
    int totalCounters = 0;
    
    for (Map.Entry<String, Long> entry : counterMap.entrySet()) {
        final String counterName = entry.getKey();
        final CounterKey counterKey = groupKey.getCounterKeyByName(counterName);
        if (counterKey == null) {
            continue;
        }
        final Long counterValue = entry.getValue();
        Bytes.putInt(buffer, pos, counterKey.getIndex());
        pos += 4;
        Bytes.putLong(buffer, pos, counterValue);
        pos += 8;
        ++totalCounters;
    }
    Bytes.putInt(buffer, totalCounterNumberPos, totalCounters);
    return pos;
}
 

private Put putInternal(Transaction tx, Put put, boolean addShadowCell) throws IOException {

        throwExceptionIfOpSetsTimerange(put);

        HBaseTransaction transaction = enforceHBaseTransactionAsParam(tx);

        final long writeTimestamp = transaction.getWriteTimestamp();

        // create put with correct ts
        final Put tsput = new Put(put.getRow(), writeTimestamp);
        propagateAttributes(put, tsput);
        Map<byte[], List<Cell>> kvs = put.getFamilyCellMap();
        for (List<Cell> kvl : kvs.values()) {
            for (Cell c : kvl) {
                CellUtils.validateCell(c, writeTimestamp);
                // Reach into keyvalue to update timestamp.
                // It's not nice to reach into keyvalue internals,
                // but we want to avoid having to copy the whole thing
                KeyValue kv = KeyValueUtil.ensureKeyValue(c);
                Bytes.putLong(kv.getValueArray(), kv.getTimestampOffset(), writeTimestamp);
                tsput.add(kv);

                if (addShadowCell) {
                    tsput.addColumn(CellUtil.cloneFamily(kv),
                            CellUtils.addShadowCellSuffixPrefix(CellUtil.cloneQualifier(kv), 0, CellUtil.cloneQualifier(kv).length),
                            kv.getTimestamp(),
                            Bytes.toBytes(kv.getTimestamp()));
                } else {
                    HBaseCellId cellId = new HBaseCellId(this,
                            CellUtil.cloneRow(kv),
                            CellUtil.cloneFamily(kv),
                            CellUtil.cloneQualifier(kv),
                            kv.getTimestamp());

                    addWriteSetElement(transaction, cellId);
                }
            }
        }
        return tsput;
    }
 

private static int encodeUnsignedLong(long v, byte[] b, int o) {
    checkForSufficientLength(b, o, Bytes.SIZEOF_LONG);
    if (v < 0) {
        throw new RuntimeException();
    }
    Bytes.putLong(b, o, v);
    return Bytes.SIZEOF_LONG;
}
 

@Override
protected void map(NullWritable key, NullWritable value,
    Context context) throws IOException, InterruptedException {

  String suffix = "/" + shortTaskId;
  byte[] row = Bytes.add(new byte[8], Bytes.toBytes(suffix));

  int BLOCK_SIZE = (int)(recordsToWrite / 100);

  for (long i = 0; i < recordsToWrite;) {
    long blockStart = i;
    for (long idxInBlock = 0;
         idxInBlock < BLOCK_SIZE && i < recordsToWrite;
         idxInBlock++, i++) {

      long byteSwapped = swapLong(i);
      Bytes.putLong(row, 0, byteSwapped);

      Put p = new Put(row);
      p.addColumn(TEST_FAMILY, TEST_QUALIFIER, HConstants.EMPTY_BYTE_ARRAY);
      if (blockStart > 0) {
        for (int j = 0; j < numBackReferencesPerRow; j++) {
          long referredRow = blockStart - BLOCK_SIZE + rand.nextInt(BLOCK_SIZE);
          Bytes.putLong(row, 0, swapLong(referredRow));
          p.addColumn(TEST_FAMILY, row, HConstants.EMPTY_BYTE_ARRAY);
        }
        refsWritten.increment(1);
      }
      rowsWritten.increment(1);
      mutator.mutate(p);

      if (i % 100 == 0) {
        context.setStatus("Written " + i + "/" + recordsToWrite + " records");
        context.progress();
      }
    }
    // End of block, flush all of them before we start writing anything
    // pointing to these!
    mutator.flush();
  }
}
 

/**
 * Create an empty byte[] representing a KeyValue
 * All lengths are preset and can be filled in later.
 * @param rlength
 * @param flength
 * @param qlength
 * @param timestamp
 * @param type
 * @param vlength
 * @return The newly created byte array.
 */
private static byte[] createEmptyByteArray(final int rlength, int flength,
    int qlength, final long timestamp, final Type type, int vlength, int tagsLength) {
  if (rlength > Short.MAX_VALUE) {
    throw new IllegalArgumentException("Row > " + Short.MAX_VALUE);
  }
  if (flength > Byte.MAX_VALUE) {
    throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE);
  }
  // Qualifier length
  if (qlength > Integer.MAX_VALUE - rlength - flength) {
    throw new IllegalArgumentException("Qualifier > " + Integer.MAX_VALUE);
  }
  RawCell.checkForTagsLength(tagsLength);
  // Key length
  long longkeylength = getKeyDataStructureSize(rlength, flength, qlength);
  if (longkeylength > Integer.MAX_VALUE) {
    throw new IllegalArgumentException("keylength " + longkeylength + " > " +
      Integer.MAX_VALUE);
  }
  int keylength = (int)longkeylength;
  // Value length
  if (vlength > HConstants.MAXIMUM_VALUE_LENGTH) { // FindBugs INT_VACUOUS_COMPARISON
    throw new IllegalArgumentException("Valuer > " +
        HConstants.MAXIMUM_VALUE_LENGTH);
  }

  // Allocate right-sized byte array.
  byte[] bytes= new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
      tagsLength)];
  // Write the correct size markers
  int pos = 0;
  pos = Bytes.putInt(bytes, pos, keylength);
  pos = Bytes.putInt(bytes, pos, vlength);
  pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
  pos += rlength;
  pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
  pos += flength + qlength;
  pos = Bytes.putLong(bytes, pos, timestamp);
  pos = Bytes.putByte(bytes, pos, type.getCode());
  pos += vlength;
  if (tagsLength > 0) {
    pos = Bytes.putAsShort(bytes, pos, tagsLength);
  }
  return bytes;
}
 

/**
 * This is a HFile block index key optimization.
 * @param leftKey
 * @param rightKey
 * @return 0 if equal, <0 if left smaller, >0 if right smaller
 * @deprecated Since 0.99.2;
 */
@Deprecated
public byte[] getShortMidpointKey(final byte[] leftKey, final byte[] rightKey) {
  if (rightKey == null) {
    throw new IllegalArgumentException("rightKey can not be null");
  }
  if (leftKey == null) {
    return Arrays.copyOf(rightKey, rightKey.length);
  }
  if (compareFlatKey(leftKey, rightKey) >= 0) {
    throw new IllegalArgumentException("Unexpected input, leftKey:" + Bytes.toString(leftKey)
      + ", rightKey:" + Bytes.toString(rightKey));
  }

  short leftRowLength = Bytes.toShort(leftKey, 0);
  short rightRowLength = Bytes.toShort(rightKey, 0);
  int leftCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + leftRowLength;
  int rightCommonLength = ROW_LENGTH_SIZE + FAMILY_LENGTH_SIZE + rightRowLength;
  int leftCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + leftCommonLength;
  int rightCommonLengthWithTSAndType = TIMESTAMP_TYPE_SIZE + rightCommonLength;
  int leftColumnLength = leftKey.length - leftCommonLengthWithTSAndType;
  int rightColumnLength = rightKey.length - rightCommonLengthWithTSAndType;
  // rows are equal
  if (leftRowLength == rightRowLength && compareRows(leftKey, ROW_LENGTH_SIZE, leftRowLength,
    rightKey, ROW_LENGTH_SIZE, rightRowLength) == 0) {
    // Compare family & qualifier together.
    int comparison = Bytes.compareTo(leftKey, leftCommonLength, leftColumnLength, rightKey,
      rightCommonLength, rightColumnLength);
    // same with "row + family + qualifier", return rightKey directly
    if (comparison == 0) {
      return Arrays.copyOf(rightKey, rightKey.length);
    }
    // "family + qualifier" are different, generate a faked key per rightKey
    byte[] newKey = Arrays.copyOf(rightKey, rightKey.length);
    Bytes.putLong(newKey, rightKey.length - TIMESTAMP_TYPE_SIZE, HConstants.LATEST_TIMESTAMP);
    Bytes.putByte(newKey, rightKey.length - TYPE_SIZE, Type.Maximum.getCode());
    return newKey;
  }
  // rows are different
  short minLength = leftRowLength < rightRowLength ? leftRowLength : rightRowLength;
  short diffIdx = 0;
  while (diffIdx < minLength
      && leftKey[ROW_LENGTH_SIZE + diffIdx] == rightKey[ROW_LENGTH_SIZE + diffIdx]) {
    diffIdx++;
  }
  byte[] newRowKey = null;
  if (diffIdx >= minLength) {
    // leftKey's row is prefix of rightKey's.
    newRowKey = new byte[diffIdx + 1];
    System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
  } else {
    int diffByte = leftKey[ROW_LENGTH_SIZE + diffIdx];
    if ((0xff & diffByte) < 0xff && (diffByte + 1) <
        (rightKey[ROW_LENGTH_SIZE + diffIdx] & 0xff)) {
      newRowKey = new byte[diffIdx + 1];
      System.arraycopy(leftKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx);
      newRowKey[diffIdx] = (byte) (diffByte + 1);
    } else {
      newRowKey = new byte[diffIdx + 1];
      System.arraycopy(rightKey, ROW_LENGTH_SIZE, newRowKey, 0, diffIdx + 1);
    }
  }
  return new KeyValue(newRowKey, null, null, HConstants.LATEST_TIMESTAMP,
    Type.Maximum).getKey();
}
 

@Override
public int encode(PositionedByteRange dst, Long val) {
  Bytes.putLong(dst.getBytes(), dst.getOffset() + dst.getPosition(), val);
  return skip(dst);
}
 

/**
 * Write instance {@code val} into buffer {@code buff}.
 */
public int encodeLong(byte[] buff, int offset, long val) {
  return Bytes.putLong(buff, offset, val);
}
 

private void decode(boolean isFirst) {
  byte flag = currentBuffer.get();
  byte type = 0;
  if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
    if (!isFirst) {
      type = current.keyBuffer[current.keyLength - Bytes.SIZEOF_BYTE];
    }
    current.keyLength = ByteBuff.readCompressedInt(currentBuffer);
  }
  if ((flag & FLAG_SAME_VALUE_LENGTH) == 0) {
    current.valueLength = ByteBuff.readCompressedInt(currentBuffer);
  }
  current.lastCommonPrefix = ByteBuff.readCompressedInt(currentBuffer);

  current.ensureSpaceForKey();

  if (current.lastCommonPrefix < Bytes.SIZEOF_SHORT) {
    // length of row is different, copy everything except family

    // copy the row size
    currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
        Bytes.SIZEOF_SHORT - current.lastCommonPrefix);
    current.rowLengthWithSize = Bytes.toShort(current.keyBuffer, 0) +
        Bytes.SIZEOF_SHORT;

    // copy the rest of row
    currentBuffer.get(current.keyBuffer, Bytes.SIZEOF_SHORT,
        current.rowLengthWithSize - Bytes.SIZEOF_SHORT);

    // copy the column family
    System.arraycopy(familyNameWithSize, 0, current.keyBuffer,
        current.rowLengthWithSize, familyNameWithSize.length);

    // copy the qualifier
    currentBuffer.get(current.keyBuffer,
        current.rowLengthWithSize + familyNameWithSize.length,
        current.keyLength - current.rowLengthWithSize -
        familyNameWithSize.length - TIMESTAMP_WITH_TYPE_LENGTH);
  } else if (current.lastCommonPrefix < current.rowLengthWithSize) {
    // we have to copy part of row and qualifier,
    // but column family is in right place

    // before column family (rest of row)
    currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
        current.rowLengthWithSize - current.lastCommonPrefix);

    // after column family (qualifier)
    currentBuffer.get(current.keyBuffer,
        current.rowLengthWithSize + familyNameWithSize.length,
        current.keyLength - current.rowLengthWithSize -
        familyNameWithSize.length - TIMESTAMP_WITH_TYPE_LENGTH);
  } else {
    // copy just the ending
    currentBuffer.get(current.keyBuffer, current.lastCommonPrefix,
        current.keyLength - TIMESTAMP_WITH_TYPE_LENGTH -
        current.lastCommonPrefix);
  }

  // timestamp
  int pos = current.keyLength - TIMESTAMP_WITH_TYPE_LENGTH;
  int timestampFitInBytes = 1 +
      ((flag & MASK_TIMESTAMP_LENGTH) >>> SHIFT_TIMESTAMP_LENGTH);
  long timestampOrDiff = ByteBuff.readLong(currentBuffer, timestampFitInBytes);
  if ((flag & FLAG_TIMESTAMP_SIGN) != 0) {
    timestampOrDiff = -timestampOrDiff;
  }
  if ((flag & FLAG_TIMESTAMP_IS_DIFF) == 0) { // it is timestamp
    current.timestamp = timestampOrDiff;
  } else { // it is diff
    current.timestamp = current.timestamp - timestampOrDiff;
  }
  Bytes.putLong(current.keyBuffer, pos, current.timestamp);
  pos += Bytes.SIZEOF_LONG;

  // type
  if ((flag & FLAG_SAME_TYPE) == 0) {
    currentBuffer.get(current.keyBuffer, pos, Bytes.SIZEOF_BYTE);
  } else if ((flag & FLAG_SAME_KEY_LENGTH) == 0) {
    current.keyBuffer[pos] = type;
  }

  current.valueOffset = currentBuffer.position();
  currentBuffer.skip(current.valueLength);

  if (includesTags()) {
    decodeTags();
  }
  if (includesMvcc()) {
    current.memstoreTS = ByteBufferUtils.readVLong(currentBuffer);
  } else {
    current.memstoreTS = 0;
  }
  current.nextKvOffset = currentBuffer.position();
}