/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.phoenix.util; import java.io.ByteArrayInputStream; import java.io.DataInput; import java.io.DataInputStream; import java.io.DataOutput; import java.io.DataOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Comparator; import java.util.List; import java.util.Set; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableUtils; import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr; import org.apache.phoenix.query.KeyRange; import org.apache.phoenix.schema.SortOrder; import org.apache.phoenix.schema.types.PDataType; import com.google.common.base.Preconditions; /** * * Byte utilities * * * @since 0.1 */ public class ByteUtil { public static final byte[] EMPTY_BYTE_ARRAY = new byte[0]; public static final ImmutableBytesPtr EMPTY_BYTE_ARRAY_PTR = new ImmutableBytesPtr( EMPTY_BYTE_ARRAY); public static final ImmutableBytesWritable EMPTY_IMMUTABLE_BYTE_ARRAY = new ImmutableBytesWritable( EMPTY_BYTE_ARRAY); public static final Comparator<ImmutableBytesPtr> BYTES_PTR_COMPARATOR = new Comparator<ImmutableBytesPtr>() { @Override public int compare(ImmutableBytesPtr o1, ImmutableBytesPtr o2) { return Bytes.compareTo(o1.get(), o1.getOffset(), o1.getLength(), o2.get(), o2.getOffset(), o2.getLength()); } }; /** * Serialize an array of byte arrays into a single byte array. Used * to pass through a set of bytes arrays as an attribute of a Scan. * Use {@link #toByteArrays(byte[], int)} to convert the serialized * byte array back to the array of byte arrays. * @param byteArrays the array of byte arrays to serialize * @return the byte array */ public static byte[] toBytes(byte[][] byteArrays) { int size = 0; for (byte[] b : byteArrays) { if (b == null) { size++; } else { size += b.length; size += WritableUtils.getVIntSize(b.length); } } TrustedByteArrayOutputStream bytesOut = new TrustedByteArrayOutputStream(size); DataOutputStream out = new DataOutputStream(bytesOut); try { for (byte[] b : byteArrays) { if (b == null) { WritableUtils.writeVInt(out, 0); } else { WritableUtils.writeVInt(out, b.length); out.write(b); } } } catch (IOException e) { throw new RuntimeException(e); // not possible } finally { try { out.close(); } catch (IOException e) { throw new RuntimeException(e); // not possible } } return bytesOut.getBuffer(); } /** * Deserialize a byte array into a set of byte arrays. Used in * coprocessor to reconstruct byte arrays from attribute value * passed through the Scan. * @param b byte array containing serialized byte arrays (created by {@link #toBytes(byte[][])}). * @param length number of byte arrays that were serialized * @return array of now deserialized byte arrays * @throws IllegalStateException if there are more than length number of byte arrays that were serialized */ public static byte[][] toByteArrays(byte[] b, int length) { return toByteArrays(b, 0, length); } public static byte[][] toByteArrays(byte[] b, int offset, int length) { ByteArrayInputStream bytesIn = new ByteArrayInputStream(b, offset, b.length - offset); DataInputStream in = new DataInputStream(bytesIn); byte[][] byteArrays = new byte[length][]; try { for (int i = 0; i < length; i++) { int bLength = WritableUtils.readVInt(in); if (bLength == 0) { byteArrays[i] = null; } else { byteArrays[i] = new byte[bLength]; int rLength = in.read(byteArrays[i], 0, bLength); assert (rLength == bLength); // For find bugs } } if (in.read() != -1) { throw new IllegalStateException("Expected only " + length + " byte arrays, but found more"); } return byteArrays; } catch (IOException e) { throw new RuntimeException(e); // not possible } finally { try { in.close(); } catch (IOException e) { throw new RuntimeException(e); // not possible } } } public static byte[] serializeVIntArray(int[] intArray) { return serializeVIntArray(intArray,intArray.length); } public static byte[] serializeVIntArray(int[] intArray, int encodedLength) { int size = WritableUtils.getVIntSize(encodedLength); for (int i = 0; i < intArray.length; i++) { size += WritableUtils.getVIntSize(intArray[i]); } int offset = 0; byte[] out = new byte[size]; offset += ByteUtil.vintToBytes(out, offset, size); for (int i = 0; i < intArray.length; i++) { offset += ByteUtil.vintToBytes(out, offset, intArray[i]); } return out; } public static void serializeVIntArray(DataOutput output, int[] intArray) throws IOException { serializeVIntArray(output, intArray, intArray.length); } /** * Allows additional stuff to be encoded in length * @param output * @param intArray * @param encodedLength * @throws IOException */ public static void serializeVIntArray(DataOutput output, int[] intArray, int encodedLength) throws IOException { WritableUtils.writeVInt(output, encodedLength); for (int i = 0; i < intArray.length; i++) { WritableUtils.writeVInt(output, intArray[i]); } } public static long[] readFixedLengthLongArray(DataInput input, int length) throws IOException { long[] longArray = new long[length]; for (int i = 0; i < length; i++) { longArray[i] = input.readLong(); } return longArray; } public static void writeFixedLengthLongArray(DataOutput output, long[] longArray) throws IOException { for (int i = 0; i < longArray.length; i++) { output.writeLong(longArray[i]); } } /** * Deserialize a byte array into a int array. * @param b byte array storing serialized vints * @return int array */ public static int[] deserializeVIntArray(byte[] b) { ByteArrayInputStream bytesIn = new ByteArrayInputStream(b); DataInputStream in = new DataInputStream(bytesIn); try { int length = WritableUtils.readVInt(in); return deserializeVIntArray(in, length); } catch (IOException e) { throw new RuntimeException(e); // not possible } finally { try { in.close(); } catch (IOException e) { throw new RuntimeException(e); // not possible } } } public static int[] deserializeVIntArray(DataInput in) throws IOException { return deserializeVIntArray(in, WritableUtils.readVInt(in)); } public static int[] deserializeVIntArray(DataInput in, int length) throws IOException { int i = 0; int[] intArray = new int[length]; while (i < length) { intArray[i++] = WritableUtils.readVInt(in); } return intArray; } /** * Deserialize a byte array into a int array. * @param b byte array storing serialized vints * @param length number of serialized vints * @return int array */ public static int[] deserializeVIntArray(byte[] b, int length) { ByteArrayInputStream bytesIn = new ByteArrayInputStream(b); DataInputStream in = new DataInputStream(bytesIn); try { return deserializeVIntArray(in,length); } catch (IOException e) { throw new RuntimeException(e); } } /** * Concatenate together one or more byte arrays * @param first first byte array * @param rest rest of byte arrays * @return newly allocated byte array that is a concatenation of all the byte arrays passed in */ public static byte[] concat(byte[] first, byte[]... rest) { int totalLength = first.length; for (byte[] array : rest) { totalLength += array.length; } byte[] result = Arrays.copyOf(first, totalLength); int offset = first.length; for (byte[] array : rest) { System.arraycopy(array, 0, result, offset, array.length); offset += array.length; } return result; } public static <T> T[] concat(T[] first, T[]... rest) { int totalLength = first.length; for (T[] array : rest) { totalLength += array.length; } T[] result = Arrays.copyOf(first, totalLength); int offset = first.length; for (T[] array : rest) { System.arraycopy(array, 0, result, offset, array.length); offset += array.length; } return result; } public static byte[] concat(SortOrder sortOrder, ImmutableBytesWritable... writables) { Preconditions.checkNotNull(sortOrder); int totalLength = 0; for (ImmutableBytesWritable writable : writables) { totalLength += writable.getLength(); } byte[] result = new byte[totalLength]; int totalOffset = 0; for (ImmutableBytesWritable array : writables) { byte[] bytes = array.get(); int offset = array.getOffset(); if (sortOrder == SortOrder.DESC) { bytes = SortOrder.invert(bytes, offset, new byte[array.getLength()], 0, array.getLength()); offset = 0; } System.arraycopy(bytes, offset, result, totalOffset, array.getLength()); totalOffset += array.getLength(); } return result; } public static int vintFromBytes(byte[] buffer, int offset) { try { return (int)Bytes.readVLong(buffer, offset); } catch (IOException e) { // Impossible throw new RuntimeException(e); } } /** * Decode a vint from the buffer pointed at to by ptr and * increment the offset of the ptr by the length of the * vint. * @param ptr a pointer to a byte array buffer * @return the decoded vint value as an int */ public static int vintFromBytes(ImmutableBytesWritable ptr) { return (int) vlongFromBytes(ptr); } /** * Decode a vint from the buffer pointed at to by ptr and * increment the offset of the ptr by the length of the * vint. * @param ptr a pointer to a byte array buffer * @return the decoded vint value as a long */ public static long vlongFromBytes(ImmutableBytesWritable ptr) { final byte [] buffer = ptr.get(); final int offset = ptr.getOffset(); byte firstByte = buffer[offset]; int len = WritableUtils.decodeVIntSize(firstByte); if (len == 1) { ptr.set(buffer, offset+1, ptr.getLength()); return firstByte; } long i = 0; for (int idx = 0; idx < len-1; idx++) { byte b = buffer[offset + 1 + idx]; i = i << 8; i = i | (b & 0xFF); } ptr.set(buffer, offset+len, ptr.getLength()); return (WritableUtils.isNegativeVInt(firstByte) ? ~i : i); } /** * Put long as variable length encoded number at the offset in the result byte array * @param vint Integer to make a vint of. * @param result buffer to put vint into * @return Vint length in bytes of vint */ public static int vintToBytes(byte[] result, int offset, final long vint) { long i = vint; if (i >= -112 && i <= 127) { result[offset] = (byte) i; return 1; } int len = -112; if (i < 0) { i ^= -1L; // take one's complement' len = -120; } long tmp = i; while (tmp != 0) { tmp = tmp >> 8; len--; } result[offset++] = (byte) len; len = (len < -120) ? -(len + 120) : -(len + 112); for (int idx = len; idx != 0; idx--) { int shiftbits = (idx - 1) * 8; long mask = 0xFFL << shiftbits; result[offset++] = (byte)((i & mask) >> shiftbits); } return len + 1; } /** * Increment the key to the next key * @param key the key to increment * @return a new byte array with the next key or null * if the key could not be incremented because it's * already at its max value. */ public static byte[] nextKey(byte[] key) { byte[] nextStartRow = new byte[key.length]; System.arraycopy(key, 0, nextStartRow, 0, key.length); if (!nextKey(nextStartRow, nextStartRow.length)) { return null; } return nextStartRow; } /** * Increment the key in-place to the next key * @param key the key to increment * @param length the length of the key * @return true if the key can be incremented and * false otherwise if the key is at its max * value. */ public static boolean nextKey(byte[] key, int length) { return nextKey(key, 0, length); } public static boolean nextKey(byte[] key, int offset, int length) { if (length == 0) { return false; } int i = offset + length - 1; while (key[i] == -1) { key[i] = 0; i--; if (i < offset) { // Change bytes back to the way they were do { key[++i] = -1; } while (i < offset + length - 1); return false; } } key[i] = (byte)(key[i] + 1); return true; } public static byte[] previousKey(byte[] key) { byte[] previousKey = new byte[key.length]; System.arraycopy(key, 0, previousKey, 0, key.length); if (!previousKey(previousKey, previousKey.length)) { return null; } return previousKey; } public static boolean previousKey(byte[] key, int length) { return previousKey(key, 0, length); } public static boolean previousKey(byte[] key, int offset, int length) { if (length == 0) { return false; } int i = offset + length - 1; while (key[i] == 0) { key[i] = -1; i--; if (i < offset) { // Change bytes back to the way they were do { key[++i] = 0; } while (i < offset + length - 1); return false; } } key[i] = (byte)(key[i] - 1); return true; } /** * Expand the key to length bytes using a null byte. */ public static byte[] fillKey(byte[] key, int length) { if(key.length > length) { throw new IllegalStateException(); } if (key.length == length) { return key; } byte[] newBound = new byte[length]; System.arraycopy(key, 0, newBound, 0, key.length); return newBound; } /** * Expand the key to length bytes using the fillByte to fill the * bytes beyond the current key length. */ public static void nullPad(ImmutableBytesWritable ptr, int length) { if(ptr.getLength() > length) { throw new IllegalStateException(); } if (ptr.getLength() == length) { return; } byte[] newBound = new byte[length]; System.arraycopy(ptr.get(), ptr.getOffset(), newBound, 0, ptr.getLength()); ptr.set(newBound); } /** * Get the size in bytes of the UTF-8 encoded CharSequence * @param sequence the CharSequence */ public static int getSize(CharSequence sequence) { int count = 0; for (int i = 0, len = sequence.length(); i < len; i++) { char ch = sequence.charAt(i); if (ch <= 0x7F) { count++; } else if (ch <= 0x7FF) { count += 2; } else if (Character.isHighSurrogate(ch)) { count += 4; ++i; } else { count += 3; } } return count; } public static boolean isInclusive(CompareOp op) { switch (op) { case LESS: case GREATER: return false; case EQUAL: case NOT_EQUAL: case LESS_OR_EQUAL: case GREATER_OR_EQUAL: return true; default: throw new RuntimeException("Unknown Compare op " + op.name()); } } public static boolean compare(CompareOp op, int compareResult) { switch (op) { case LESS: return compareResult < 0; case LESS_OR_EQUAL: return compareResult <= 0; case EQUAL: return compareResult == 0; case NOT_EQUAL: return compareResult != 0; case GREATER_OR_EQUAL: return compareResult >= 0; case GREATER: return compareResult > 0; default: throw new RuntimeException("Unknown Compare op " + op.name()); } } /** * Given an ImmutableBytesWritable, returns the payload part of the argument as an byte array. */ public static byte[] copyKeyBytesIfNecessary(ImmutableBytesWritable ptr) { if (ptr.getOffset() == 0 && ptr.getLength() == ptr.get().length) { return ptr.get(); } return ptr.copyBytes(); } public static KeyRange getKeyRange(byte[] key, CompareOp op, PDataType type) { switch (op) { case EQUAL: return type.getKeyRange(key, true, key, true); case GREATER: return type.getKeyRange(key, false, KeyRange.UNBOUND, false); case GREATER_OR_EQUAL: return type.getKeyRange(key, true, KeyRange.UNBOUND, false); case LESS: return type.getKeyRange(KeyRange.UNBOUND, false, key, false); case LESS_OR_EQUAL: return type.getKeyRange(KeyRange.UNBOUND, false, key, true); default: throw new IllegalArgumentException("Unknown operator " + op); } } public static boolean contains(Collection<byte[]> keys, byte[] key) { for (byte[] k : keys) { if (Arrays.equals(k, key)) { return true; } } return false; } public static boolean contains(List<ImmutableBytesPtr> keys, ImmutableBytesPtr key) { for (ImmutableBytesPtr k : keys) { if (key.compareTo(k) == 0) { return true; } } return false; } public static boolean match(Set<byte[]> keys, Set<byte[]> keys2) { if (keys == keys2) return true; if (keys == null || keys2 == null) return false; int size = keys.size(); if (keys2.size() != size) return false; for (byte[] k : keys) { if (!contains(keys2, k)) { return false; } } return true; } public static byte[] calculateTheClosestNextRowKeyForPrefix(byte[] rowKeyPrefix) { // Essentially we are treating it like an 'unsigned very very long' and doing +1 manually. // Search for the place where the trailing 0xFFs start int offset = rowKeyPrefix.length; while (offset > 0) { if (rowKeyPrefix[offset - 1] != (byte) 0xFF) { break; } offset--; } if (offset == 0) { // We got an 0xFFFF... (only FFs) stopRow value which is // the last possible prefix before the end of the table. // So set it to stop at the 'end of the table' return HConstants.EMPTY_END_ROW; } // Copy the right length of the original byte[] newStopRow = Arrays.copyOfRange(rowKeyPrefix, 0, offset); // And increment the last one newStopRow[newStopRow.length - 1]++; return newStopRow; } public static byte[][] splitArrayBySeparator(byte[] src, byte separator){ List<Integer> separatorLocations = new ArrayList<Integer>(); for (int k = 0; k < src.length; k++){ if (src[k] == separator){ separatorLocations.add(k); } } byte[][] dst = new byte[separatorLocations.size() +1][]; int previousSepartor = -1; for (int j = 0; j < separatorLocations.size(); j++){ int separatorLocation = separatorLocations.get(j); dst[j] = Bytes.copy(src, previousSepartor +1, separatorLocation- previousSepartor -1); previousSepartor = separatorLocation; } if (previousSepartor < src.length){ dst[separatorLocations.size()] = Bytes.copy(src, previousSepartor +1, src.length - previousSepartor -1); } return dst; } }