Java Code Examples for org.apache.hadoop.hbase.util.Bytes#putBytes()

The following examples show how to use org.apache.hadoop.hbase.util.Bytes#putBytes() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: hbase   File: WALCellCodec.java    License: Apache License 2.0 6 votes vote down vote up
private int readIntoArray(byte[] to, int offset, Dictionary dict) throws IOException {
  byte status = (byte)in.read();
  if (status == Dictionary.NOT_IN_DICTIONARY) {
    // status byte indicating that data to be read is not in dictionary.
    // if this isn't in the dictionary, we need to add to the dictionary.
    int length = StreamUtils.readRawVarint32(in);
    IOUtils.readFully(in, to, offset, length);
    dict.addEntry(to, offset, length);
    return length;
  } else {
    // the status byte also acts as the higher order byte of the dictionary entry.
    short dictIdx = StreamUtils.toShort(status, (byte)in.read());
    byte[] entry = dict.getEntry(dictIdx);
    if (entry == null) {
      throw new IOException("Missing dictionary entry for index " + dictIdx);
    }
    // now we write the uncompressed value.
    Bytes.putBytes(to, offset, entry, 0, entry.length);
    return entry.length;
  }
}
 
Example 2
Source Project: hbase   File: ServerRpcConnection.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Send the response for connection header
 */
private void responseConnectionHeader(RPCProtos.ConnectionHeaderResponse.Builder chrBuilder)
    throws FatalConnectionException {
  // Response the connection header if Crypto AES is enabled
  if (!chrBuilder.hasCryptoCipherMeta()) return;
  try {
    byte[] connectionHeaderResBytes = chrBuilder.build().toByteArray();
    // encrypt the Crypto AES cipher meta data with sasl server, and send to client
    byte[] unwrapped = new byte[connectionHeaderResBytes.length + 4];
    Bytes.putBytes(unwrapped, 0, Bytes.toBytes(connectionHeaderResBytes.length), 0, 4);
    Bytes.putBytes(unwrapped, 4, connectionHeaderResBytes, 0, connectionHeaderResBytes.length);
    byte[] wrapped = saslServer.wrap(unwrapped, 0, unwrapped.length);
    BufferChain bc;
    try (ByteBufferOutputStream response = new ByteBufferOutputStream(wrapped.length + 4);
        DataOutputStream out = new DataOutputStream(response)) {
      out.writeInt(wrapped.length);
      out.write(wrapped);
      bc = new BufferChain(response.getByteBuffer());
    }
    doRespond(() -> bc);
  } catch (IOException ex) {
    throw new UnsupportedCryptoException(ex.getMessage(), ex);
  }
}
 
Example 3
public static ProjectedValueTuple mergeProjectedValue(ProjectedValueTuple dest, KeyValueSchema destSchema, ValueBitSet destBitSet,
		Tuple src, KeyValueSchema srcSchema, ValueBitSet srcBitSet, int offset) throws IOException {
	ImmutableBytesWritable destValue = new ImmutableBytesWritable(dest.getProjectedValue());
	destBitSet.clear();
	destBitSet.or(destValue);
	int origDestBitSetLen = dest.getBitSetLength();
	ImmutableBytesWritable srcValue = new ImmutableBytesWritable();
	decodeProjectedValue(src, srcValue);
	srcBitSet.clear();
	srcBitSet.or(srcValue);
	int origSrcBitSetLen = srcBitSet.getEstimatedLength();
	for (int i = 0; i < srcBitSet.getMaxSetBit(); i++) {
		if (srcBitSet.get(i)) {
			destBitSet.set(offset + i);
		}
	}
	int destBitSetLen = destBitSet.getEstimatedLength();
	byte[] merged = new byte[destValue.getLength() - origDestBitSetLen + srcValue.getLength() - origSrcBitSetLen + destBitSetLen];
	int o = Bytes.putBytes(merged, 0, destValue.get(), destValue.getOffset(), destValue.getLength() - origDestBitSetLen);
	o = Bytes.putBytes(merged, o, srcValue.get(), srcValue.getOffset(), srcValue.getLength() - origSrcBitSetLen);
	destBitSet.toBytes(merged, o);
	ImmutableBytesWritable keyPtr = dest.getKeyPtr();
    return new ProjectedValueTuple(keyPtr.get(), keyPtr.getOffset(), keyPtr.getLength(), dest.getTimestamp(), merged, destBitSetLen);
}
 
Example 4
Source Project: hbase   File: ParseFilter.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Extracts a simple filter expression from the filter string given by the user
* <p>
* A simpleFilterExpression is of the form: FilterName('arg', 'arg', 'arg')
* The user given filter string can have many simpleFilterExpressions combined
* using operators.
* <p>
* This function extracts a simpleFilterExpression from the
* larger filterString given the start offset of the simpler expression
* <p>
* @param filterStringAsByteArray filter string given by the user
* @param filterExpressionStartOffset start index of the simple filter expression
* @return byte array containing the simple filter expression
*/
 public byte [] extractFilterSimpleExpression (byte [] filterStringAsByteArray,
                                               int filterExpressionStartOffset)
   throws CharacterCodingException {
   int quoteCount = 0;
   for (int i=filterExpressionStartOffset; i<filterStringAsByteArray.length; i++) {
     if (filterStringAsByteArray[i] == ParseConstants.SINGLE_QUOTE) {
       if (isQuoteUnescaped(filterStringAsByteArray, i)) {
         quoteCount ++;
       } else {
         // To skip the next quote that has been escaped
         i++;
       }
     }
     if (filterStringAsByteArray[i] == ParseConstants.RPAREN && (quoteCount %2 ) == 0) {
       byte [] filterSimpleExpression = new byte [i - filterExpressionStartOffset + 1];
       Bytes.putBytes(filterSimpleExpression, 0, filterStringAsByteArray,
                      filterExpressionStartOffset, i-filterExpressionStartOffset + 1);
       return filterSimpleExpression;
     }
   }
   throw new IllegalArgumentException("Incorrect Filter String");
 }
 
Example 5
Source Project: hbase   File: ParseFilter.java    License: Apache License 2.0 6 votes vote down vote up
/**
* Returns the filter name given a simple filter expression
* <p>
* @param filterStringAsByteArray a simple filter expression
* @return name of filter in the simple filter expression
*/
 public static byte [] getFilterName (byte [] filterStringAsByteArray) {
   int filterNameStartIndex = 0;
   int filterNameEndIndex = 0;

   for (int i=filterNameStartIndex; i<filterStringAsByteArray.length; i++) {
     if (filterStringAsByteArray[i] == ParseConstants.LPAREN ||
         filterStringAsByteArray[i] == ParseConstants.WHITESPACE) {
       filterNameEndIndex = i;
       break;
     }
   }

   if (filterNameEndIndex == 0) {
     throw new IllegalArgumentException("Incorrect Filter Name");
   }

   byte [] filterName = new byte[filterNameEndIndex - filterNameStartIndex];
   Bytes.putBytes(filterName, 0, filterStringAsByteArray, 0,
                  filterNameEndIndex - filterNameStartIndex);
   return filterName;
 }
 
Example 6
Source Project: hbase   File: TagUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static byte[] concatTags(byte[] tags, Cell cell) {
  int cellTagsLen = cell.getTagsLength();
  if (cellTagsLen == 0) {
    // If no Tags, return early.
    return tags;
  }
  byte[] b = new byte[tags.length + cellTagsLen];
  int pos = Bytes.putBytes(b, 0, tags, 0, tags.length);
  if (cell instanceof ByteBufferExtendedCell) {
    ByteBufferUtils.copyFromBufferToArray(b, ((ByteBufferExtendedCell) cell).getTagsByteBuffer(),
        ((ByteBufferExtendedCell) cell).getTagsPosition(), pos, cellTagsLen);
  } else {
    Bytes.putBytes(b, pos, cell.getTagsArray(), cell.getTagsOffset(), cellTagsLen);
  }
  return b;
}
 
Example 7
public static byte[] generateOutputTableRowKey(long ts, byte[] indexTableName, byte[] dataRowKey ) {
    byte[] keyPrefix = Bytes.toBytes(Long.toString(ts));
    byte[] rowKey;
    int targetOffset = 0;
    // The row key for the output table : timestamp | index table name | data row key
    rowKey = new byte[keyPrefix.length + ROW_KEY_SEPARATOR_BYTE.length + indexTableName.length +
        ROW_KEY_SEPARATOR_BYTE.length + dataRowKey.length];
    Bytes.putBytes(rowKey, targetOffset, keyPrefix, 0, keyPrefix.length);
    targetOffset += keyPrefix.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, indexTableName, 0, indexTableName.length);
    targetOffset += indexTableName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, dataRowKey, 0, dataRowKey.length);
    return rowKey;
}
 
Example 8
/**
 * Extract byte array for column specified by colIdx.
 * 
 * @param lineBytes
 * @param parsed
 * @param colIdx
 * @return
 */
protected byte[] getInputColBytes(byte[] lineBytes,
		ImportTsv.TsvParser.ParsedLine parsed, int colIdx) {
	if (colIdx >= columnTypes.length)
		return null;
	int colOffset = parsed.getColumnOffset(colIdx);
	int colLen = parsed.getColumnLength(colIdx);
	byte[] colBytes = new byte[colLen];
	Bytes.putBytes(colBytes, 0, lineBytes, colOffset, colLen);
	return colBytes;
}
 
Example 9
Source Project: hraven   File: TestTaskKey.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSerialization() {
  TaskKeyConverter conv = new TaskKeyConverter();

  TaskKey key1 = new TaskKey(
      new JobKey("[email protected]", "testuser", "app", 1234L, "job_20120101000000_1111"), "m_001");
  assertEquals("[email protected]", key1.getCluster());
  assertEquals("testuser", key1.getUserName());
  assertEquals("app", key1.getAppId());
  assertEquals(1234L, key1.getRunId());
  assertEquals("job_20120101000000_1111", key1.getJobId().getJobIdString());
  assertEquals("m_001", key1.getTaskId());

  byte[] key1Bytes = conv.toBytes(key1);
  TaskKey key2 = conv.fromBytes(key1Bytes);
  assertKey(key1, key2);

  TaskKey key3 = conv.fromBytes( conv.toBytes(key2) );
  assertKey(key1, key3);

  // test with a run ID containing the separator
  long now = System.currentTimeMillis();
  byte[] encoded = Bytes.toBytes(Long.MAX_VALUE - now);
  // replace last byte with separator and reconvert to long
  Bytes.putBytes(encoded, encoded.length-Constants.SEP_BYTES.length,
      Constants.SEP_BYTES, 0, Constants.SEP_BYTES.length);
  long badId = Long.MAX_VALUE - Bytes.toLong(encoded);
  LOG.info("Bad run ID is " + badId);

  TaskKey badKey1 = new TaskKey(
      new JobKey(key1.getQualifiedJobId(), key1.getUserName(), key1.getAppId(), badId),
      key1.getTaskId());
  byte[] badKeyBytes = conv.toBytes(badKey1);
  TaskKey badKey2 = conv.fromBytes(badKeyBytes);
  assertKey(badKey1, badKey2);
}
 
Example 10
Source Project: hbase   File: ZKMetadata.java    License: Apache License 2.0 5 votes vote down vote up
public static byte[] appendMetaData(byte[] id, byte[] data) {
  if (data == null || data.length == 0) {
    return data;
  }
  byte[] salt = Bytes.toBytes(ThreadLocalRandom.current().nextLong());
  int idLength = id.length + salt.length;
  byte[] newData = new byte[MAGIC_SIZE + ID_LENGTH_SIZE + idLength + data.length];
  int pos = 0;
  pos = Bytes.putByte(newData, pos, MAGIC);
  pos = Bytes.putInt(newData, pos, idLength);
  pos = Bytes.putBytes(newData, pos, id, 0, id.length);
  pos = Bytes.putBytes(newData, pos, salt, 0, salt.length);
  pos = Bytes.putBytes(newData, pos, data, 0, data.length);
  return newData;
}
 
Example 11
Source Project: hbase   File: TestLRUDictionary.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBasic() {
  Random rand = new Random();
  byte[] testBytes = new byte[10];
  rand.nextBytes(testBytes);

  // Verify that our randomly generated array doesn't exist in the dictionary
  assertEquals(-1, testee.findEntry(testBytes, 0, testBytes.length));

  // now since we looked up an entry, we should have added it to the
  // dictionary, so it isn't empty

  assertFalse(isDictionaryEmpty(testee));

  // Check if we can find it using findEntry
  short t = testee.findEntry(testBytes, 0, testBytes.length);

  // Making sure we do find what we're looking for
  assertTrue(t != -1);

  byte[] testBytesCopy = new byte[20];

  Bytes.putBytes(testBytesCopy, 10, testBytes, 0, testBytes.length);

  // copy byte arrays, make sure that we check that equal byte arrays are
  // equal without just checking the reference
  assertEquals(testee.findEntry(testBytesCopy, 10, testBytes.length), t);

  // make sure the entry retrieved is the same as the one put in
  assertTrue(Arrays.equals(testBytes, testee.getEntry(t)));

  testee.clear();

  // making sure clear clears the dictionary
  assertTrue(isDictionaryEmpty(testee));
}
 
Example 12
Source Project: hbase   File: KeyValue.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write KeyValue format into a byte array.
 * @param row row key
 * @param roffset row offset
 * @param rlength row length
 * @param family family name
 * @param foffset family offset
 * @param flength family length
 * @param qualifier column qualifier
 * @param qoffset qualifier offset
 * @param qlength qualifier length
 * @param timestamp version timestamp
 * @param type key type
 * @param value column value
 * @param voffset value offset
 * @param vlength value length
 * @return The newly created byte array.
 */
private static byte [] createByteArray(final byte [] row, final int roffset,
    final int rlength, final byte [] family, final int foffset, int flength,
    final byte [] qualifier, final int qoffset, int qlength,
    final long timestamp, final Type type,
    final byte [] value, final int voffset,
    int vlength, byte[] tags, int tagsOffset, int tagsLength) {

  checkParameters(row, rlength, family, flength, qlength, vlength);
  RawCell.checkForTagsLength(tagsLength);
  // Allocate right-sized byte array.
  int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength);
  byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength,
    tagsLength)];
  // Write key, value and key row length.
  int pos = 0;
  pos = Bytes.putInt(bytes, pos, keyLength);
  pos = Bytes.putInt(bytes, pos, vlength);
  pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff));
  pos = Bytes.putBytes(bytes, pos, row, roffset, rlength);
  pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff));
  if(flength != 0) {
    pos = Bytes.putBytes(bytes, pos, family, foffset, flength);
  }
  if(qlength != 0) {
    pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength);
  }
  pos = Bytes.putLong(bytes, pos, timestamp);
  pos = Bytes.putByte(bytes, pos, type.getCode());
  if (value != null && value.length > 0) {
    pos = Bytes.putBytes(bytes, pos, value, voffset, vlength);
  }
  // Add the tags after the value part
  if (tagsLength > 0) {
    pos = Bytes.putAsShort(bytes, pos, tagsLength);
    pos = Bytes.putBytes(bytes, pos, tags, tagsOffset, tagsLength);
  }
  return bytes;
}
 
Example 13
private static byte[] generateResultTableRowKey(long ts, byte[] indexTableName,  byte [] regionName,
                                                byte[] startRow, byte[] stopRow) {
    byte[] keyPrefix = Bytes.toBytes(Long.toString(ts));
    int targetOffset = 0;
    // The row key for the result table : timestamp | index table name | datable table region name |
    //                                    scan start row | scan stop row
    byte[] rowKey = new byte[keyPrefix.length + ROW_KEY_SEPARATOR_BYTE.length + indexTableName.length +
        ROW_KEY_SEPARATOR_BYTE.length + regionName.length + ROW_KEY_SEPARATOR_BYTE.length +
        startRow.length + ROW_KEY_SEPARATOR_BYTE.length + stopRow.length];
    Bytes.putBytes(rowKey, targetOffset, keyPrefix, 0, keyPrefix.length);
    targetOffset += keyPrefix.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, indexTableName, 0, indexTableName.length);
    targetOffset += indexTableName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, regionName, 0, regionName.length);
    targetOffset += regionName.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, startRow, 0, startRow.length);
    targetOffset += startRow.length;
    Bytes.putBytes(rowKey, targetOffset, ROW_KEY_SEPARATOR_BYTE, 0, ROW_KEY_SEPARATOR_BYTE.length);
    targetOffset += ROW_KEY_SEPARATOR_BYTE.length;
    Bytes.putBytes(rowKey, targetOffset, stopRow, 0, stopRow.length);
    return rowKey;
}
 
Example 14
Source Project: hbase   File: Tag.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Copies the tag's value bytes to the given byte array
 * @param tag The Tag
 * @param out The byte array where to copy the Tag value.
 * @param offset The offset within 'out' array where to copy the Tag value.
 */
public static void copyValueTo(Tag tag, byte[] out, int offset) {
  if (tag.hasArray()) {
    Bytes.putBytes(out, offset, tag.getValueArray(), tag.getValueOffset(), tag.getValueLength());
  } else {
    ByteBufferUtils.copyFromBufferToArray(out, tag.getValueByteBuffer(), tag.getValueOffset(),
      offset, tag.getValueLength());
  }
}
 
Example 15
Source Project: hbase   File: ArrayBackedTag.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Format for a tag :
 * {@code <length of tag - 2 bytes><type code - 1 byte><tag>} tag length is serialized
 * using 2 bytes only but as this will be unsigned, we can have max tag length of
 * (Short.MAX_SIZE * 2) +1. It includes 1 byte type length and actual tag bytes length.
 */
public ArrayBackedTag(byte tagType, byte[] tag) {
  int tagLength = tag.length + TYPE_LENGTH_SIZE;
  if (tagLength > MAX_TAG_LENGTH) {
    throw new IllegalArgumentException(
        "Invalid tag data being passed. Its length can not exceed " + MAX_TAG_LENGTH);
  }
  length = TAG_LENGTH_SIZE + tagLength;
  bytes = new byte[length];
  int pos = Bytes.putAsShort(bytes, 0, tagLength);
  pos = Bytes.putByte(bytes, pos, tagType);
  Bytes.putBytes(bytes, pos, tag, 0, tag.length);
  this.type = tagType;
}
 
Example 16
Source Project: hbase   File: LRUDictionary.java    License: Apache License 2.0 5 votes vote down vote up
private short put(byte[] array, int offset, int length, boolean copy) {
  if (copy) {
    // We copy the bytes we want, otherwise we might be holding references to
    // massive arrays in our dictionary (or those arrays might change)
    byte[] stored = new byte[length];
    Bytes.putBytes(stored, 0, array, offset, length);
    return putInternal(stored);
  } else {
    return putInternal(array);
  }
}
 
Example 17
@Test
public void testEndToEnd() throws Exception {
  String cf = "f";

  Table ht =
      TEST_UTIL.createTable(TableName.valueOf(table), Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i0 = 0; i0 < firstPartCardinality; i0++) {

    for (int i1 = 0; i1 < secondPartCardinality; i1++) {

      for (int i2 = 0; i2 < thirdPartCardinality; i2++) {
        byte[] rk = new byte[10];

        ByteBuffer buf = ByteBuffer.wrap(rk);
        buf.clear();
        buf.putShort((short) i0);
        buf.putInt(i1);
        buf.putInt(i2);
        for (int c = 0; c < colQualifiersTotal; c++) {
          byte[] cq = new byte[4];
          Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

          Put p = new Put(rk);
          p.setDurability(Durability.SKIP_WAL);
          p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
          ht.put(p);
        }
      }
    }
  }

  TEST_UTIL.flush();

  // test passes
  runTest1(ht);
  runTest2(ht);

}
 
Example 18
@SuppressWarnings("deprecation")
@Test
public void testFilterList() throws Exception {
  String cf = "f";
  Table ht =
      TEST_UTIL.createTable(TableName.valueOf(name.getMethodName()), Bytes.toBytes(cf), Integer.MAX_VALUE);

  // 10 byte row key - (2 bytes 4 bytes 4 bytes)
  // 4 byte qualifier
  // 4 byte value

  for (int i1 = 0; i1 < 5; i1++) {
    for (int i2 = 0; i2 < 5; i2++) {
      byte[] rk = new byte[10];

      ByteBuffer buf = ByteBuffer.wrap(rk);
      buf.clear();
      buf.putShort((short) 2);
      buf.putInt(i1);
      buf.putInt(i2);

      // Each row contains 5 columns
      for (int c = 0; c < 5; c++) {
        byte[] cq = new byte[4];
        Bytes.putBytes(cq, 0, Bytes.toBytes(c), 0, 4);

        Put p = new Put(rk);
        p.setDurability(Durability.SKIP_WAL);
        p.addColumn(Bytes.toBytes(cf), cq, Bytes.toBytes(c));
        ht.put(p);
        LOG.info("Inserting: rk: " + Bytes.toStringBinary(rk) + " cq: "
            + Bytes.toStringBinary(cq));
      }
    }
  }

  TEST_UTIL.flush();

  // test passes if we get back 5 KV's (1 row)
  runTest(ht, 5);

}
 
Example 19
Source Project: hbase   File: KeyValue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setTimestamp(long ts) {
  Bytes.putBytes(this.bytes, this.getTimestampOffset(), Bytes.toBytes(ts), 0, Bytes.SIZEOF_LONG);
}
 
Example 20
Source Project: hbase   File: KeyValue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void setTimestamp(byte[] ts) {
  Bytes.putBytes(this.bytes, this.getTimestampOffset(), ts, 0, Bytes.SIZEOF_LONG);
}