Java Code Examples for org.apache.arrow.vector.FieldVector#getValidityBuffer()

The following examples show how to use org.apache.arrow.vector.FieldVector#getValidityBuffer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 6 votes vote down vote up 
private void setNextNValuesInVector(
    int typeWidth, NullabilityHolder nullabilityHolder,
    ValuesAsBytesReader valuesReader, int bufferIdx, FieldVector vector, int numValues) {
  ArrowBuf validityBuffer = vector.getValidityBuffer();
  if (currentValue == maxDefLevel) {
    ByteBuffer buffer = valuesReader.getBuffer(numValues * typeWidth);
    vector.getDataBuffer().setBytes(bufferIdx * typeWidth, buffer);
    nullabilityHolder.setNotNulls(bufferIdx, numValues);
    if (setArrowValidityVector) {
      for (int i = 0; i < numValues; i++) {
        BitVectorHelper.setValidityBitToOne(validityBuffer, bufferIdx + i);
      }
    }
  } else {
    setNulls(nullabilityHolder, bufferIdx, numValues, validityBuffer);
  }
}
Example 2
Source File: PartitionToLoadSpilledData.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up 
/**
 * This is done after reading every spilled batch so that
 * we can correctly identify length of data read in every batch
 * and the length doesn't end up being cumulative which will then
 * lead to incorrect results, segfaults etc.
 *
 * It is probably not necessary to zero out the buffers again
 * Overwriting the contents should be fine as long as we
 * correctly identify the length read in each iteration.
 *
 * Zeroing out is likely an expensive operation so unless
 * we figure out it is absolutely necessary for correctness,
 * we will just reset the reader/writer indexes.
 */
public void reset() {
  recordsInBatch = 0;
  fixedKeyColPivotedData.readerIndex(0);
  fixedKeyColPivotedData.writerIndex(0);
  variableKeyColPivotedData.readerIndex(0);
  variableKeyColPivotedData.writerIndex(0);
  for (FieldVector vector : postSpillAccumulatorVectors) {
    final ArrowBuf validityBuffer = vector.getValidityBuffer();
    final ArrowBuf dataBuffer = vector.getDataBuffer();
    validityBuffer.readerIndex(0);
    validityBuffer.writerIndex(0);
    dataBuffer.readerIndex(0);
    dataBuffer.writerIndex(0);
    validityBuffer.setZero(0, validityBuffer.capacity());
    dataBuffer.setZero(0, dataBuffer.capacity());
    vector.setValueCount(0);
  }
}
Example 3
Source File: BaseSingleAccumulator.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up 
private void resetFirstAccumulatorVector() {
  Preconditions.checkArgument(accumulators.length == 1, "Error: incorrect number of batches in accumulator");
  final FieldVector vector = accumulators[0];
  Preconditions.checkArgument(vector != null, "Error: expecting a valid accumulator");
  final ArrowBuf validityBuffer = vector.getValidityBuffer();
  final ArrowBuf dataBuffer = vector.getDataBuffer();
  validityBuffer.readerIndex(0);
  validityBuffer.writerIndex(0);
  dataBuffer.readerIndex(0);
  dataBuffer.writerIndex(0);
  initialize(vector);
  vector.setValueCount(0);
}
Example 4
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfTimestampMillis(final FieldVector vector, final int startOffset, final int typeWidth,
                                       final int numValsToRead, NullabilityHolder nullabilityHolder,
                                       ValuesAsBytesReader valuesReader) {
  int bufferIdx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int numValues = Math.min(left, this.currentCount);
    switch (mode) {
      case RLE:
        ArrowBuf validityBuffer = vector.getValidityBuffer();
        if (currentValue == maxDefLevel) {
          for (int i = 0; i < numValues; i++) {
            vector.getDataBuffer().setLong(bufferIdx * typeWidth, valuesReader.readLong() * 1000);
          }
          nullabilityHolder.setNotNulls(bufferIdx, numValues);
          if (setArrowValidityVector) {
            for (int i = 0; i < numValues; i++) {
              BitVectorHelper.setValidityBitToOne(validityBuffer, bufferIdx + i);
            }
          }
        } else {
          setNulls(nullabilityHolder, bufferIdx, numValues, validityBuffer);
        }
        bufferIdx += numValues;
        break;
      case PACKED:
        for (int i = 0; i < numValues; i++) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            vector.getDataBuffer().setLong(bufferIdx * typeWidth, valuesReader.readLong() * 1000);
            nullabilityHolder.setNotNull(bufferIdx);
            if (setArrowValidityVector) {
              BitVectorHelper.setValidityBitToOne(vector.getValidityBuffer(), bufferIdx);
            }
          } else {
            setNull(nullabilityHolder, bufferIdx, vector.getValidityBuffer());
          }
          bufferIdx++;
        }
        break;
    }
    left -= numValues;
    currentCount -= numValues;
  }
}
Example 5
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfDictionaryEncodedLongs(
    final FieldVector vector,
    final int startOffset,
    final int typeWidth,
    final int numValsToRead,
    NullabilityHolder nullabilityHolder,
    VectorizedDictionaryEncodedParquetValuesReader dictionaryEncodedValuesReader,
    Dictionary dict) {
  int idx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int numValues = Math.min(left, this.currentCount);
    ArrowBuf validityBuffer = vector.getValidityBuffer();
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          dictionaryEncodedValuesReader.readBatchOfDictionaryEncodedLongs(vector,
              idx, numValues, dict, nullabilityHolder, typeWidth);
        } else {
          setNulls(nullabilityHolder, idx, numValues, validityBuffer);
        }
        idx += numValues;
        break;
      case PACKED:
        for (int i = 0; i < numValues; i++) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            vector.getDataBuffer().setLong(idx * typeWidth,
                dict.decodeToLong(dictionaryEncodedValuesReader.readInteger()));
            nullabilityHolder.setNotNull(idx);
            if (setArrowValidityVector) {
              BitVectorHelper.setValidityBitToOne(vector.getValidityBuffer(), idx);
            }
          } else {
            setNull(nullabilityHolder, idx, validityBuffer);
          }
          idx++;
        }
        break;
    }
    left -= numValues;
    currentCount -= numValues;
  }
}
Example 6
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfDictionaryEncodedTimestampMillis(
    final FieldVector vector,
    final int startOffset,
    final int typeWidth,
    final int numValsToRead,
    NullabilityHolder nullabilityHolder,
    VectorizedDictionaryEncodedParquetValuesReader dictionaryEncodedValuesReader,
    Dictionary dict) {
  int idx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int numValues = Math.min(left, this.currentCount);
    ArrowBuf validityBuffer = vector.getValidityBuffer();
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          dictionaryEncodedValuesReader.readBatchOfDictionaryEncodedTimestampMillis(vector,
              idx, numValues, dict, nullabilityHolder, typeWidth);
        } else {
          setNulls(nullabilityHolder, idx, numValues, validityBuffer);
        }
        idx += numValues;
        break;
      case PACKED:
        for (int i = 0; i < numValues; i++) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            vector.getDataBuffer().setLong(idx * typeWidth,
                dict.decodeToLong(dictionaryEncodedValuesReader.readInteger()) * 1000);
            nullabilityHolder.setNotNull(idx);
            if (setArrowValidityVector) {
              BitVectorHelper.setValidityBitToOne(vector.getValidityBuffer(), idx);
            }
          } else {
            setNull(nullabilityHolder, idx, validityBuffer);
          }
          idx++;
        }
        break;
    }
    left -= numValues;
    currentCount -= numValues;
  }
}
Example 7
Source File: VectorizedParquetDefinitionLevelReader.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
public void readBatchOfDictionaryEncodedFloats(
    final FieldVector vector,
    final int startOffset,
    final int typeWidth,
    final int numValsToRead,
    NullabilityHolder nullabilityHolder,
    VectorizedDictionaryEncodedParquetValuesReader dictionaryEncodedValuesReader,
    Dictionary dict) {
  int idx = startOffset;
  int left = numValsToRead;
  while (left > 0) {
    if (this.currentCount == 0) {
      this.readNextGroup();
    }
    int num = Math.min(left, this.currentCount);
    ArrowBuf validityBuffer = vector.getValidityBuffer();
    switch (mode) {
      case RLE:
        if (currentValue == maxDefLevel) {
          dictionaryEncodedValuesReader.readBatchOfDictionaryEncodedFloats(vector, idx,
              num, dict, nullabilityHolder, typeWidth);
        } else {
          setNulls(nullabilityHolder, idx, num, validityBuffer);
        }
        idx += num;
        break;
      case PACKED:
        for (int i = 0; i < num; i++) {
          if (packedValuesBuffer[packedValuesBufferIdx++] == maxDefLevel) {
            vector.getDataBuffer()
                .setFloat(idx * typeWidth, dict.decodeToFloat(dictionaryEncodedValuesReader.readInteger()));
            nullabilityHolder.setNotNull(idx);
            if (setArrowValidityVector) {
              BitVectorHelper.setValidityBitToOne(vector.getValidityBuffer(), idx);
            }
          } else {
            setNull(nullabilityHolder, idx, validityBuffer);
          }
          idx++;
        }
        break;
    }
    left -= num;
    currentCount -= num;
  }
}