Java Code Examples for org.apache.hadoop.hbase.util.Bytes#SIZEOF_INT

@Override
public Result deserialize(Result mutation) throws IOException {
  int totalBuffer = in.readInt();
  if (totalBuffer == 0) {
    return Result.EMPTY_RESULT;
  }
  byte[] buf = new byte[totalBuffer];
  readChunked(in, buf, 0, totalBuffer);
  List<Cell> kvs = new ArrayList<>();
  int offset = 0;
  while (offset < totalBuffer) {
    int keyLength = Bytes.toInt(buf, offset);
    offset += Bytes.SIZEOF_INT;
    kvs.add(new KeyValue(buf, offset, keyLength));
    offset += keyLength;
  }
  return Result.create(kvs);
}
 

private int countMapHeapSize() {
    int size = 0;
    if (this.valueVsCount.size() > 0) {
        for (ImmutableBytesPtr key : this.valueVsCount.keySet()) {
            size += SizedUtil.MAP_ENTRY_SIZE + // entry
                    Bytes.SIZEOF_INT + // key size
                    key.getLength() + SizedUtil.ARRAY_SIZE; // value size
        }
    } else {
        // Initially when the getSize() is called, we dont have any entries in the map so as to
        // tell the exact heap need. Let us approximate the #entries
        SizedUtil.sizeOfMap(DEFAULT_ESTIMATED_DISTINCT_VALUES,
                SizedUtil.IMMUTABLE_BYTES_PTR_SIZE, Bytes.SIZEOF_INT);
    }
    return size;
}
 

private int getInt(int index, int itemIndex) {
  ByteBuffer item = items[itemIndex];
  int offsetInItem = index - this.itemBeginPos[itemIndex];
  int remainingLen = item.limit() - offsetInItem;
  if (remainingLen >= Bytes.SIZEOF_INT) {
    return ByteBufferUtils.toInt(item, offsetInItem);
  }
  if (items.length - 1 == itemIndex) {
    // means cur item is the last one and we wont be able to read a int. Throw exception
    throw new BufferUnderflowException();
  }
  int l = 0;
  for (int i = 0; i < Bytes.SIZEOF_INT; i++) {
    l <<= 8;
    l ^= get(index + i) & 0xFF;
  }
  return l;
}
 

/**
 * Inserts / Replaces cache element in the currently loaded page. Direct access via mapped page map
 * 
 * @param key
 * @param value
 */
public void addElement(byte[] spilledValue, ImmutableBytesPtr key, byte[] value) {

    // put Element into map
    pageMap.put(key, value);
    // Update bloom filter
    bFilter.put(key.copyBytesIfNecessary());
    // track current Map size to prevent Buffer overflows
    if (spilledValue != null) {
        // if previous key was present, just add the size difference
        totalResultSize += Math.max(0, value.length - (spilledValue.length));
    } else {
        // Add new size information
        totalResultSize += (value.length + Bytes.SIZEOF_INT);
    }

    dirtyPage = true;
}
 

private void pageIn() throws IndexOutOfBoundsException {
    if (!pagedIn) {
        // Map the memory region
        MappedByteBuffer buffer = spillFile.getPage(pageIndex);
        int numElements = buffer.getInt();
        for (int i = 0; i < numElements; i++) {
            int kvSize = buffer.getInt();
            byte[] data = new byte[kvSize];
            buffer.get(data, 0, kvSize);
            try {
                pageMap.put(SpillManager.getKey(data), data);
                totalResultSize += (data.length + Bytes.SIZEOF_INT);
            } catch (IOException ioe) {
                // Error during key access on spilled resource
                // TODO rework error handling
                throw new RuntimeException(ioe);
            }
        }
        pagedIn = true;
        dirtyPage = false;
    }
}
 

public SpillMap(SpillFile file, int thresholdBytes, int estValueSize, SpillableGroupByCache.QueryCache cache)
        throws IOException {
    this.thresholdBytes = thresholdBytes - Bytes.SIZEOF_INT;
    this.pageInserts = thresholdBytes / estValueSize;
    this.spillFile = file;
    this.cache = cache;

    // Init the e-hashing directory structure
    globalDepth = 1;
    directory = new MappedByteBufferMap[(1 << globalDepth)];

    for (int i = 0; i < directory.length; i++) {
        // Create an empty bucket list
        directory[i] = new MappedByteBufferMap(i, this.thresholdBytes, pageInserts, file);
        directory[i].flushBuffer();
    }
    directory[0].pageIn();
    curMapBufferIndex = 0;
}
 

/**
 * Adds a new entry to this block index chunk.
 *
 * @param firstKey the first key in the block pointed to by this entry
 * @param blockOffset the offset of the next-level block pointed to by this
 *          entry
 * @param onDiskDataSize the on-disk data of the block pointed to by this
 *          entry, including header size
 * @param curTotalNumSubEntries if this chunk is the root index chunk under
 *          construction, this specifies the current total number of
 *          sub-entries in all leaf-level chunks, including the one
 *          corresponding to the second-level entry being added.
 */
void add(byte[] firstKey, long blockOffset, int onDiskDataSize,
    long curTotalNumSubEntries) {
  // Record the offset for the secondary index
  secondaryIndexOffsetMarks.add(curTotalNonRootEntrySize);
  curTotalNonRootEntrySize += SECONDARY_INDEX_ENTRY_OVERHEAD
      + firstKey.length;

  curTotalRootSize += Bytes.SIZEOF_LONG + Bytes.SIZEOF_INT
      + WritableUtils.getVIntSize(firstKey.length) + firstKey.length;

  blockKeys.add(firstKey);
  blockOffsets.add(blockOffset);
  onDiskDataSizes.add(onDiskDataSize);

  if (curTotalNumSubEntries != -1) {
    numSubEntriesAt.add(curTotalNumSubEntries);

    // Make sure the parallel arrays are in sync.
    if (numSubEntriesAt.size() != blockKeys.size()) {
      throw new IllegalStateException("Only have key/value count " +
          "stats for " + numSubEntriesAt.size() + " block index " +
          "entries out of " + blockKeys.size());
    }
  }
}
 

@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  if (v < 0) {
    throw newIllegalDataException();
  }
  Bytes.putInt(b, o, v);
  return Bytes.SIZEOF_INT;
}
 

/**
 * Test a bunch of random allocations
 */
@Test
public void testLABRandomAllocation() {
  Random rand = new Random();
  MemStoreLAB mslab = new MemStoreLABImpl();
  int expectedOff = 0;
  ByteBuffer lastBuffer = null;
  int lastChunkId = -1;
  // 100K iterations by 0-1K alloc -> 50MB expected
  // should be reasonable for unit test and also cover wraparound
  // behavior
  for (int i = 0; i < 100000; i++) {
    int valSize = rand.nextInt(3);
    KeyValue kv = new KeyValue(rk, cf, q, new byte[valSize]);
    int size = kv.getSerializedSize();
    ByteBufferKeyValue newKv = (ByteBufferKeyValue) mslab.copyCellInto(kv);
    if (newKv.getBuffer() != lastBuffer) {
      // since we add the chunkID at the 0th offset of the chunk and the
      // chunkid is an int we need to account for those 4 bytes
      expectedOff = Bytes.SIZEOF_INT;
      lastBuffer = newKv.getBuffer();
      int chunkId = newKv.getBuffer().getInt(0);
      assertTrue("chunkid should be different", chunkId != lastChunkId);
      lastChunkId = chunkId;
    }
    assertEquals(expectedOff, newKv.getOffset());
    assertTrue("Allocation overruns buffer",
        newKv.getOffset() + size <= newKv.getBuffer().capacity());
    expectedOff += size;
  }
}
 

@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  b[o + 0] = (byte) ((v >> 24) ^ 0x80); // Flip sign bit so that INTEGER is binary comparable
  b[o + 1] = (byte) (v >> 16);
  b[o + 2] = (byte) (v >> 8);
  b[o + 3] = (byte) v;
  return Bytes.SIZEOF_INT;
}
 

/**
 * Compose the final index row key.
 * <p>
 * This is faster than adding each value independently as we can just build a single a array and copy everything
 * over once.
 * 
 * @param pk
 *            primary key of the original row
 * @param length
 *            total number of bytes of all the values that should be added
 * @param values
 *            to use when building the key
 */
public static byte[] composeRowKey(byte[] pk, int length, List<ColumnEntry> values) {
    final int numColumnEntries = values.size() * Bytes.SIZEOF_INT;
    // now build up expected row key, each of the values, in order, followed by the PK and then some
    // info about lengths so we can deserialize each value
    //
    // output = length of values + primary key + column entries + length of each column entry + number of column entries
    byte[] output = new byte[length + pk.length + numColumnEntries + Bytes.SIZEOF_INT];
    int pos = 0;
    int[] lengths = new int[values.size()];
    int i = 0;
    for (ColumnEntry entry : values) {
        byte[] v = entry.value;
        // skip doing the copy attempt, if we don't need to
        if (v.length != 0) {
            System.arraycopy(v, 0, output, pos, v.length);
            pos += v.length;
        }
        lengths[i++] = v.length;
    }

    // add the primary key to the end of the row key
    System.arraycopy(pk, 0, output, pos, pk.length);
    pos += pk.length;

    // add the lengths as suffixes so we can deserialize the elements again
    for (int l : lengths) {
        byte[] serializedLength = Bytes.toBytes(l);
        System.arraycopy(serializedLength, 0, output, pos, Bytes.SIZEOF_INT);
        pos += Bytes.SIZEOF_INT;
    }

    // and the last integer is the number of values
    byte[] serializedNumValues = Bytes.toBytes(values.size());
    System.arraycopy(serializedNumValues, 0, output, pos, Bytes.SIZEOF_INT);
    // Just in case we serialize more in the rowkey in the future..
    pos += Bytes.SIZEOF_INT;

    return output;
}
 

static int getSerializedOffset(byte[] bytes, int arrayIndex, boolean useShort, int indexOffset, byte serializationVersion) {
	int offset;
       if (useShort) {
           offset = indexOffset + (Bytes.SIZEOF_SHORT * arrayIndex);
           return Bytes.toShort(bytes, offset, Bytes.SIZEOF_SHORT)
                   + (serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_VERSION
                           || serializationVersion == PArrayDataType.IMMUTABLE_SERIALIZATION_V2 ? 0
                                   : Short.MAX_VALUE);
       } else {
           offset = indexOffset + (Bytes.SIZEOF_INT * arrayIndex);
           return Bytes.toInt(bytes, offset, Bytes.SIZEOF_INT);
       }
}
 

private int sizeof(ImmutableBytesWritable[] sortKeys) {
    int size = Bytes.SIZEOF_INT;
    if (sortKeys != null) {
        for (ImmutableBytesWritable sortKey : sortKeys) {
            if (sortKey != null) {
                size += sortKey.getLength();
            }
            size += Bytes.SIZEOF_INT;
        }
    }
    return size;
}
 

@Override
public int encodeInt(int v, byte[] b, int o) {
  checkForSufficientLength(b, o, Bytes.SIZEOF_INT);
  if (v < 0) {
    throw newIllegalDataException();
  }
  Bytes.putInt(b, o, v);
  return Bytes.SIZEOF_INT;
}
 

public static int getByteSize(int capacity) {
    if (capacity <= BitSet.BITS_PER_BYTE) {
        return Bytes.SIZEOF_BYTE;
    } else if (capacity <= BitSet.BITS_PER_SHORT) {
        return Bytes.SIZEOF_SHORT;
    } else if (capacity <= BitSet.BITS_PER_INT) {
        return Bytes.SIZEOF_INT;
    } else if (capacity <= BitSet.BITS_PER_LONG) {
        return Bytes.SIZEOF_LONG;
    } else {
        int nLongs = (capacity-1) / BitSet.BITS_PER_LONG + 1;
        return nLongs * Bytes.SIZEOF_LONG;
    }
}
 

@Override
public PositionedByteRange putInt(int val) {
    ensureCapacity(position + Bytes.SIZEOF_INT);
    putInt(position, val);
    position += Bytes.SIZEOF_INT;
    return this;
}
 

private static long sizeof(ImmutableBytesWritable[] sortKeys) {
    long size = Bytes.SIZEOF_INT;
    if (sortKeys != null) {
        for (ImmutableBytesWritable sortKey : sortKeys) {
            if (sortKey != null) {
                size += sortKey.getLength();
            }
            size += Bytes.SIZEOF_INT;
        }
    }
    return size;
}
 

@Override
protected long sizeOf(Tuple e) {
    KeyValue kv = PhoenixKeyValueUtil.maybeCopyCell(e.getValue(0));
    return Bytes.SIZEOF_INT * 2 + kv.getLength();
}
 

@Override
public Integer getByteSize() {
  return Bytes.SIZEOF_INT;
}
 

@Override
public void writeInt(int i) {
  checkSizeAndGrow(Bytes.SIZEOF_INT);
  Bytes.putInt(this.buf, this.pos, i);
  this.pos += Bytes.SIZEOF_INT;
}