Java Code Examples for org.apache.spark.unsafe.Platform#getInt()

public static int binarySearchInts(ByteBufferReader reader, int count, int key) throws IOException {
    int from = 0;
    int to = count;

    --to;

    byte[] valBuffer = new byte[4];

    while (from <= to) {
        int mid = from + to >>> 1;
        reader.read(mid << 2, valBuffer, 0, 4);
        int midVal = Platform.getInt(valBuffer, Platform.BYTE_ARRAY_OFFSET);
        if (midVal < key) {
            from = mid + 1;
        } else if (midVal > key) {
            to = mid - 1;
        } else {
            return mid;
        }
    }

    return -(from + 1);
}
 

@Override
public Location next() {
    if (recordsInPage == 0) {
        advanceToNextPage();
    }
    numRecords--;
    if (currentPage != null) {
        int totalLength = Platform.getInt(pageBaseObject, offsetInPage);
        loc.with(currentPage, offsetInPage);
        offsetInPage += 4 + totalLength;
        recordsInPage--;
        return loc;
    } else {
        assert (reader != null);
        if (!reader.hasNext()) {
            advanceToNextPage();
        }
        try {
            reader.loadNext();
        } catch (IOException e) {
            try {
                reader.close();
            } catch (IOException e2) {
                logger.error("Error while closing spill reader", e2);
            }
            // Scala iterator does not handle exception
            Platform.throwException(e);
        }
        loc.with(reader.getBaseObject(), reader.getBaseOffset(), reader.getRecordLength());
        return loc;
    }
}
 

private void updateAddressesAndSizes(final Object base, final long offset) {
    long position = offset;
    final int totalLength = Platform.getInt(base, position);
    position += 4;
    keyLength = Platform.getInt(base, position);
    position += 4;
    valueLength = totalLength - keyLength - 4;

    keyMemoryLocation.setObjAndOffset(base, position);

    position += keyLength;
    valueMemoryLocation.setObjAndOffset(base, position);
}
 

/**
 * This is only used for spilling
 */
private Location with(Object base, long offset, int length) {
    this.isDefined = true;
    this.memoryPage = null;
    keyLength = Platform.getInt(base, offset);
    valueLength = length - 4 - keyLength;
    keyMemoryLocation.setObjAndOffset(base, offset + 4);
    valueMemoryLocation.setObjAndOffset(base, offset + 4 + keyLength);
    return this;
}
 

@Override
public boolean next() throws IOException {
    try {
        if (underlying.hasNext()) {
            underlying.loadNext();

            Object baseObj = underlying.getBaseObject();
            long recordOffset = underlying.getBaseOffset();
            int recordLen = underlying.getRecordLength();

            // Note that recordLen = keyLen + valueLen + 4 bytes (for the keyLen itself)
            int keyLen = Platform.getInt(baseObj, recordOffset);
            int valueLen = recordLen - keyLen - 4;
            key.pointTo(baseObj, recordOffset + 4, keyLen);
            value.pointTo(baseObj, recordOffset + 4 + keyLen, valueLen);

            return true;
        } else {
            key = null;
            value = null;
            cleanupResources();
            return false;
        }
    } catch (IOException e) {
        cleanupResources();
        throw e;
    }
}
 

@Override
public void loadNext() {
    // This pointer points to a 4-byte record length, followed by the record's bytes
    final long recordPointer = array.get(position);
    baseObject = memoryManager.getPage(recordPointer);
    baseOffset = memoryManager.getOffsetInPage(recordPointer) + 4;  // Skip over record length
    recordLength = Platform.getInt(baseObject, baseOffset - 4);
    keyPrefix = array.get(position + 1);
    position += 2;
}
 

private int readInt(int pos) throws IOException {
    reader.read(pos, intValBuffer, 0, 4);
    return Platform.getInt(intValBuffer, Platform.BYTE_ARRAY_OFFSET);
}
 

default int readInt(long position) throws IOException {
    byte[] valBuffer = new byte[4];
    read(position, valBuffer, 0, 4);
    return Platform.getInt(valBuffer, Platform.BYTE_ARRAY_OFFSET);
}
 

public long spill(long numBytes) throws IOException {
    synchronized (this) {
        if (!destructive || dataPages.size() == 1) {
            return 0L;
        }

        // TODO: use existing ShuffleWriteMetrics
        ShuffleWriteMetrics writeMetrics = new ShuffleWriteMetrics();

        long released = 0L;
        while (dataPages.size() > 0) {
            MemoryBlock block = dataPages.getLast();
            // The currentPage is used, cannot be released
            if (block == currentPage) {
                break;
            }

            Object base = block.getBaseObject();
            long offset = block.getBaseOffset();
            int numRecords = Platform.getInt(base, offset);
            offset += 4;
            final UnsafeSorterSpillWriter writer =
                    new UnsafeSorterSpillWriter(numRecords);
            while (numRecords > 0) {
                int length = Platform.getInt(base, offset);
                writer.write(base, offset + 4, length, 0);
                offset += 4 + length;
                numRecords--;
            }
            writer.close();
            spillWriters.add(writer);

            dataPages.removeLast();
            released += block.size();
            freePage(block);

            if (released >= numBytes) {
                break;
            }
        }

        return released;
    }
}
 

@Override
public int getInt(int ordinal) {
    assertIndexIsValid(ordinal);
    return Platform.getInt(baseObject, getFieldOffset(ordinal));
}
 

public static int getInt(Object base, long addr) {
    return Platform.getInt(base, addr);
}