Java Code Examples for org.apache.hadoop.hbase.util.Bytes#SIZEOF_INT

The following examples show how to use org.apache.hadoop.hbase.util.Bytes#SIZEOF_INT . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: hbase   File: ResultSerialization.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Result deserialize(Result mutation) throws IOException {
  int totalBuffer = in.readInt();
  if (totalBuffer == 0) {
    return Result.EMPTY_RESULT;
  }
  byte[] buf = new byte[totalBuffer];
  readChunked(in, buf, 0, totalBuffer);
  List<Cell> kvs = new ArrayList<>();
  int offset = 0;
  while (offset < totalBuffer) {
    int keyLength = Bytes.toInt(buf, offset);
    offset += Bytes.SIZEOF_INT;
    kvs.add(new KeyValue(buf, offset, keyLength));
    offset += keyLength;
  }
  return Result.create(kvs);
}
 
Example 2
private int countMapHeapSize() {
    int size = 0;
    if (this.valueVsCount.size() > 0) {
        for (ImmutableBytesPtr key : this.valueVsCount.keySet()) {
            size += SizedUtil.MAP_ENTRY_SIZE + // entry
                    Bytes.SIZEOF_INT + // key size
                    key.getLength() + SizedUtil.ARRAY_SIZE; // value size
        }
    } else {
        // Initially when the getSize() is called, we dont have any entries in the map so as to
        // tell the exact heap need. Let us approximate the #entries
        SizedUtil.sizeOfMap(DEFAULT_ESTIMATED_DISTINCT_VALUES,
                SizedUtil.IMMUTABLE_BYTES_PTR_SIZE, Bytes.SIZEOF_INT);
    }
    return size;
}
 
Example 3
Source Project: hbase   File: MultiByteBuff.java    License: Apache License 2.0 6 votes vote down vote up
private int getInt(int index, int itemIndex) {
  ByteBuffer item = items[itemIndex];
  int offsetInItem = index - this.itemBeginPos[itemIndex];
  int remainingLen = item.limit() - offsetInItem;
  if (remainingLen >= Bytes.SIZEOF_INT) {
    return ByteBufferUtils.toInt(item, offsetInItem);
  }
  if (items.length - 1 == itemIndex) {
    // means cur item is the last one and we wont be able to read a int. Throw exception
    throw new BufferUnderflowException();
  }
  int l = 0;
  for (int i = 0; i < Bytes.SIZEOF_INT; i++) {
    l <<= 8;
    l ^= get(index + i) & 0xFF;
  }
  return l;
}
 
Example 4
Source Project: hbase   File: HFileBlockIndex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds a new entry to this block index chunk.
 *
 * @param firstKey the first key in the block pointed to by this entry
 * @param blockOffset the offset of the next-level block pointed to by this
 *          entry
 * @param onDiskDataSize the on-disk data of the block pointed to by this
 *          entry, including header size
 * @param curTotalNumSubEntries if this chunk is the root index chunk under
 *          construction, this specifies the current total number of
 *          sub-entries in all leaf-level chunks, including the one
 *          corresponding to the second-level entry being added.
 */
void add(byte[] firstKey, long blockOffset, int onDiskDataSize,
    long curTotalNumSubEntries) {
  // Record the offset for the secondary index
  secondaryIndexOffsetMarks.add(curTotalNonRootEntrySize);
  curTotalNonRootEntrySize += SECONDARY_INDEX_ENTRY_OVERHEAD
      + firstKey.length;

  curTotalRootSize += Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT
      + WritableUtils.getVIntSize(firstKey.length) + firstKey.length;

  blockKeys.add(firstKey);
  blockOffsets.add(blockOffset);
  onDiskDataSizes.add(onDiskDataSize);

  if (curTotalNumSubEntries != -1) {
    numSubEntriesAt.add(curTotalNumSubEntries);

    // Make sure the parallel arrays are in sync.
    if (numSubEntriesAt.size() != blockKeys.size()) {
      throw new IllegalStateException("Only have key/value count " +
          "stats for " + numSubEntriesAt.size() + " block index " +
          "entries out of " + blockKeys.size());
    }
  }
}
 
Example 5
public SpillMap(SpillFile file, int thresholdBytes, int estValueSize, SpillableGroupByCache.QueryCache cache)
        throws IOException {
    this.thresholdBytes = thresholdBytes - Bytes.SIZEOF_INT;
    this.pageInserts = thresholdBytes / estValueSize;
    this.spillFile = file;
    this.cache = cache;

    // Init the e-hashing directory structure
    globalDepth = 1;
    directory = new MappedByteBufferMap[(1 << globalDepth)];

    for (int i = 0; i < directory.length; i++) {
        // Create an empty bucket list
        directory[i] = new MappedByteBufferMap(i, this.thresholdBytes, pageInserts, file);
        directory[i].flushBuffer();
    }
    directory[0].pageIn();
    curMapBufferIndex = 0;
}
 
Example 6
Source Project: phoenix   File: SpillMap.java    License: Apache License 2.0 6 votes vote down vote up
private void pageIn() throws IndexOutOfBoundsException {
    if (!pagedIn) {
        // Map the memory region
        MappedByteBuffer buffer = spillFile.getPage(pageIndex);
        int numElements = buffer.getInt();
        for (int i = 0; i < numElements; i++) {
            int kvSize = buffer.getInt();
            byte[] data = new byte[kvSize];
            buffer.get(data, 0, kvSize);
            try {
                pageMap.put(SpillManager.getKey(data), data);
                totalResultSize += (data.length + Bytes.SIZEOF_INT);
            } catch (IOException ioe) {
                // Error during key access on spilled resource
                // TODO rework error handling
                throw new RuntimeException(ioe);
            }
        }
        pagedIn = true;
        dirtyPage = false;
    }
}
 
Example 7
Source Project: phoenix   File: SpillMap.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Inserts / Replaces cache element in the currently loaded page. Direct access via mapped page map
 * 
 * @param key
 * @param value
 */
public void addElement(byte[] spilledValue, ImmutableBytesPtr key, byte[] value) {

    // put Element into map
    pageMap.put(key, value);
    // Update bloom filter
    bFilter.put(key.copyBytesIfNecessary());
    // track current Map size to prevent Buffer overflows
    if (spilledValue != null) {
        // if previous key was present, just add the size difference
        totalResultSize += Math.max(0, value.length - (spilledValue.length));
    } else {
        // Add new size information
        totalResultSize += (value.length + Bytes.SIZEOF_INT);
    }

    dirtyPage = true;
}
 
Example 8
Source Project: hbase   File: TestMemStoreLAB.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Test a bunch of random allocations
 */
@Test
public void testLABRandomAllocation() {
  Random rand = new Random();
  MemStoreLAB mslab = new MemStoreLABImpl();
  int expectedOff = 0;
  ByteBuffer lastBuffer = null;
  int lastChunkId = -1;
  // 100K iterations by 0-1K alloc -> 50MB expected
  // should be reasonable for unit test and also cover wraparound
  // behavior
  for (int i = 0; i < 100000; i++) {
    int valSize = rand.nextInt(3);
    KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]);
    int size = kv.getSerializedSize();
    ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv);
    if (newKv.getBuffer() != lastBuffer) {
      // since we add the chunkID at the 0th offset of the chunk and the
      // chunkid is an int we need to account for those 4 bytes
      expectedOff = Bytes.SIZEOF_INT;
      lastBuffer = newKv.getBuffer();
      int chunkId = newKv.getBuffer().getInt(0);
      assertTrue("chunkid should be different", chunkId != lastChunkId);
      lastChunkId = chunkId;
    }
    assertEquals(expectedOff, newKv.getOffset());
    assertTrue("Allocation overruns buffer",
        newKv.getOffset() + size <= newKv.getBuffer().capacity());
    expectedOff += size;
  }
}
 
Example 9
Source Project: phoenix   File: OrderedResultIterator.java    License: Apache License 2.0 5 votes vote down vote up
private static long sizeof(ImmutableBytesWritable[] sortKeys) {
    long size = Bytes.SIZEOF_INT;
    if (sortKeys != null) {
        for (ImmutableBytesWritable sortKey : sortKeys) {
            if (sortKey != null) {
                size += sortKey.getLength();
            }
            size += Bytes.SIZEOF_INT;
        }
    }
    return size;
}
 
Example 10
@Override
public PositionedByteRange putInt(int val) {
    ensureCapacity(position + Bytes.SIZEOF_INT);
    putInt(position, val);
    position += Bytes.SIZEOF_INT;
    return this;
}
 
Example 11
Source Project: phoenix   File: BitSet.java    License: Apache License 2.0 5 votes vote down vote up
public static int getByteSize(int capacity) {
    if (capacity <= BitSet.BITS_PER_BYTE) {
        return Bytes.SIZEOF_BYTE;
    } else if (capacity <= BitSet.BITS_PER_SHORT) {
        return Bytes.SIZEOF_SHORT;
    } else if (capacity <= BitSet.BITS_PER_INT) {
        return Bytes.SIZEOF_INT;
    } else if (capacity <= BitSet.BITS_PER_LONG) {
        return Bytes.SIZEOF_LONG;
    } else {
        int nLongs = (capacity-1) / BitSet.BITS_PER_LONG + 1;
        return nLongs * Bytes.SIZEOF_LONG;
    }
}
 
Example 12
Source Project: phoenix   File: PUnsignedInt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  if (v < 0) {
    throw newIllegalDataException();
  }
  Bytes.putInt(b, o, v);
  return Bytes.SIZEOF_INT;
}
 
Example 13
private int sizeof(ImmutableBytesWritable[] sortKeys) {
    int size = Bytes.SIZEOF_INT;
    if (sortKeys != null) {
        for (ImmutableBytesWritable sortKey : sortKeys) {
            if (sortKey != null) {
                size += sortKey.getLength();
            }
            size += Bytes.SIZEOF_INT;
        }
    }
    return size;
}
 
Example 14
Source Project: phoenix   File: PArrayDataType.java    License: Apache License 2.0 5 votes vote down vote up
static int getSerializedOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset, byte serializationVersion) {
	int offset;
       if (useShort) {
           offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
           return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT)
                   + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION
                           || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2 ? 0
                                   : Short.MAX_VALUE);
       } else {
           offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
           return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
       }
}
 
Example 15
Source Project: phoenix   File: CoveredColumnIndexCodec.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Compose the final index row key.
 * <p>
 * This is faster than adding each value independently as we can just build a single a array and copy everything
 * over once.
 * 
 * @param pk
 *            primary key of the original row
 * @param length
 *            total number of bytes of all the values that should be added
 * @param values
 *            to use when building the key
 */
public static byte[] composeRowKey(byte[] pk, int length, List<ColumnEntry> values) {
    final int numColumnEntries = values.size() * Bytes.SIZEOF_INT;
    // now build up expected row key, each of the values, in order, followed by the PK and then some
    // info about lengths so we can deserialize each value
    //
    // output = length of values + primary key + column entries + length of each column entry + number of column entries
    byte[] output = new byte[length + pk.length + numColumnEntries + Bytes.SIZEOF_INT];
    int pos = 0;
    int[] lengths = new int[values.size()];
    int i = 0;
    for (ColumnEntry entry : values) {
        byte[] v = entry.value;
        // skip doing the copy attempt, if we don't need to
        if (v.length != 0) {
            System.arraycopy(v, 0, output, pos, v.length);
            pos += v.length;
        }
        lengths[i++] = v.length;
    }

    // add the primary key to the end of the row key
    System.arraycopy(pk, 0, output, pos, pk.length);
    pos += pk.length;

    // add the lengths as suffixes so we can deserialize the elements again
    for (int l : lengths) {
        byte[] serializedLength = Bytes.toBytes(l);
        System.arraycopy(serializedLength, 0, output, pos, Bytes.SIZEOF_INT);
        pos += Bytes.SIZEOF_INT;
    }

    // and the last integer is the number of values
    byte[] serializedNumValues = Bytes.toBytes(values.size());
    System.arraycopy(serializedNumValues, 0, output, pos, Bytes.SIZEOF_INT);
    // Just in case we serialize more in the rowkey in the future..
    pos += Bytes.SIZEOF_INT;

    return output;
}
 
Example 16
Source Project: phoenix   File: PInteger.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  b[o + 0] = (byte) ((v >> 24) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable
  b[o + 1] = (byte) (v >> 16);
  b[o + 2] = (byte) (v >> 8);
  b[o + 3] = (byte) v;
  return Bytes.SIZEOF_INT;
}
 
Example 17
Source Project: phoenix   File: PUnsignedInt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  if (v < 0) {
    throw newIllegalDataException();
  }
  Bytes.putInt(b, o, v);
  return Bytes.SIZEOF_INT;
}
 
Example 18
Source Project: hbase   File: ByteArrayOutputStream.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void writeInt(int i) {
  checkSizeAndGrow(Bytes.SIZEOF_INT);
  Bytes.putInt(this.buf, this.pos, i);
  this.pos += Bytes.SIZEOF_INT;
}
 
Example 19
Source Project: phoenix   File: PInteger.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Integer getByteSize() {
  return Bytes.SIZEOF_INT;
}
 
Example 20
Source Project: phoenix   File: BufferedTupleQueue.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected long sizeOf(Tuple e) {
    KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}