Java Code Examples for org.apache.spark.unsafe.Platform#putLong()

The following examples show how to use org.apache.spark.unsafe.Platform#putLong() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 6 votes vote down vote up 
/**
 * UnsafeArrayWriter doesn't have a binary form that lets the user pass an
 * offset and length, so I've added one here. It is the minor tweak of the
 * UnsafeArrayWriter.write(int, byte[]) method.
 * @param holder the BufferHolder where the bytes are being written
 * @param writer the UnsafeArrayWriter
 * @param ordinal the element that we are writing into
 * @param input the input bytes
 * @param offset the first byte from input
 * @param length the number of bytes to write
 */
static void write(BufferHolder holder, UnsafeArrayWriter writer, int ordinal,
                  byte[] input, int offset, int length) {
  final int roundedSize = ByteArrayMethods.roundNumberOfBytesToNearestWord(length);

  // grow the global buffer before writing data.
  holder.grow(roundedSize);

  if ((length & 0x07) > 0) {
    Platform.putLong(holder.buffer, holder.cursor + ((length >> 3) << 3), 0L);
  }

  // Write the bytes to the variable length portion.
  Platform.copyMemory(input, Platform.BYTE_ARRAY_OFFSET + offset,
      holder.buffer, holder.cursor, length);

  writer.setOffsetAndSize(ordinal, holder.cursor, length);

  // move the cursor forward.
  holder.cursor += roundedSize;
}
Example 2
Source File: SparkOrcReader.java    From iceberg with Apache License 2.0 6 votes vote down vote up 
int writeMap(MapColumnVector v, int row) {
  int offset = (int) v.offsets[row];
  int length = (int) v.lengths[row];
  int start = holder.cursor;
  // save room for the key size
  final int KEY_SIZE_BYTES = 8;
  holder.grow(KEY_SIZE_BYTES);
  holder.cursor += KEY_SIZE_BYTES;
  // serialize the keys
  childWriter.initialize(holder, length, keySize);
  for(int c=0; c < length; ++c) {
    keyConvert.convert(childWriter, c, v.keys, offset + c);
  }
  // store the serialized size of the keys
  Platform.putLong(holder.buffer, start, holder.cursor - start - KEY_SIZE_BYTES);
  // serialize the values
  childWriter.initialize(holder, length, valueSize);
  for(int c=0; c < length; ++c) {
    valueConvert.convert(childWriter, c, v.values, offset + c);
  }
  return start;
}
Example 3
Source File: CodegenExamples.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  MapData value = isNull ? null : (i.getMap(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeMapData) {

      final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      final ArrayData keys = value.keyArray();
      final ArrayData values = value.valueArray();

      // preserve 8 bytes to write the key array numBytes later.
      holder.grow(8);
      holder.cursor += 8;

      // Remember the current cursor so that we can write numBytes of key array later.
      final int tmpCursor1 = holder.cursor;


      if (keys instanceof UnsafeArrayData) {

        final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes1);
        ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes1;

      } else {
        final int numElements = keys.numElements();
        arrayWriter.initialize(holder, numElements, 8);

        for (int index = 0; index < numElements; index++) {
          if (keys.isNullAt(index)) {
            arrayWriter.setNull(index);
          } else {
            final UTF8String element = keys.getUTF8String(index);
            arrayWriter.write(index, element);
          }
        }
      }

      // Write the numBytes of key array into the first 8 bytes.
      Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);


      if (values instanceof UnsafeArrayData) {

        final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes2);
        ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes2;

      } else {
        final int numElements1 = values.numElements();
        arrayWriter1.initialize(holder, numElements1, 8);

        for (int index1 = 0; index1 < numElements1; index1++) {
          if (values.isNullAt(index1)) {
            arrayWriter1.setNull(index1);
          } else {
            final UTF8String element1 = values.getUTF8String(index1);
            arrayWriter1.write(index1, element1);
          }
        }
      }

    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
Example 4
Source File: CodegenExamples.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public UnsafeRow apply(InternalRow i) {
  holder.reset();

  rowWriter.zeroOutNullBytes();


  boolean isNull = i.isNullAt(0);
  MapData value = isNull ? null : (i.getMap(0));
  if (isNull) {
    rowWriter.setNullAt(0);
  } else {
    // Remember the current cursor so that we can calculate how many bytes are
    // written later.
    final int tmpCursor = holder.cursor;

    if (value instanceof UnsafeMapData) {

      final int sizeInBytes = ((UnsafeMapData) value).getSizeInBytes();
      // grow the global buffer before writing data.
      holder.grow(sizeInBytes);
      ((UnsafeMapData) value).writeToMemory(holder.buffer, holder.cursor);
      holder.cursor += sizeInBytes;

    } else {
      final ArrayData keys = value.keyArray();
      final ArrayData values = value.valueArray();

      // preserve 8 bytes to write the key array numBytes later.
      holder.grow(8);
      holder.cursor += 8;

      // Remember the current cursor so that we can write numBytes of key array later.
      final int tmpCursor1 = holder.cursor;


      if (keys instanceof UnsafeArrayData) {

        final int sizeInBytes1 = ((UnsafeArrayData) keys).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes1);
        ((UnsafeArrayData) keys).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes1;

      } else {
        final int numElements = keys.numElements();
        arrayWriter.initialize(holder, numElements, 8);

        for (int index = 0; index < numElements; index++) {
          if (keys.isNullAt(index)) {
            arrayWriter.setNull(index);
          } else {
            final UTF8String element = keys.getUTF8String(index);
            arrayWriter.write(index, element);
          }
        }
      }

      // Write the numBytes of key array into the first 8 bytes.
      Platform.putLong(holder.buffer, tmpCursor1 - 8, holder.cursor - tmpCursor1);


      if (values instanceof UnsafeArrayData) {

        final int sizeInBytes2 = ((UnsafeArrayData) values).getSizeInBytes();
        // grow the global buffer before writing data.
        holder.grow(sizeInBytes2);
        ((UnsafeArrayData) values).writeToMemory(holder.buffer, holder.cursor);
        holder.cursor += sizeInBytes2;

      } else {
        final int numElements1 = values.numElements();
        arrayWriter1.initialize(holder, numElements1, 8);

        for (int index1 = 0; index1 < numElements1; index1++) {
          if (values.isNullAt(index1)) {
            arrayWriter1.setNull(index1);
          } else {
            final UTF8String element1 = values.getUTF8String(index1);
            arrayWriter1.write(index1, element1);
          }
        }
      }

    }

    rowWriter.setOffsetAndSize(0, tmpCursor, holder.cursor - tmpCursor);
  }
  result.setTotalSize(holder.totalSize());
  return result;
}
Example 5
Source File: UnsafeRow.java    From indexr with Apache License 2.0 4 votes vote down vote up 
@Override
public void setUniformVal(int ordinal, long value) {
    assertIndexIsValid(ordinal);
    Platform.putLong(baseObject, getFieldOffset(ordinal), value);
}
Example 6
Source File: UnsafeRow.java    From indexr with Apache License 2.0 4 votes vote down vote up 
@Override
public void setLong(int ordinal, long value) {
    assertIndexIsValid(ordinal);
    Platform.putLong(baseObject, getFieldOffset(ordinal), value);
}
Example 7
Source File: UnsafeUtil.java    From indexr with Apache License 2.0 4 votes vote down vote up 
public static void setLong(Object base, long addr, long v) {
    Platform.putLong(base, addr, v);
}