org.apache.parquet.bytes.ByteBufferAllocator Java Examples

public ByteStreamSplitValuesWriter(int elementSizeInBytes, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  if (elementSizeInBytes <= 0) {
    throw new ParquetEncodingException(String.format("Element byte size is invalid: %d", elementSizeInBytes));
  }
  this.numStreams = elementSizeInBytes;
  this.elementSizeInBytes = elementSizeInBytes;
  this.byteStreams = new CapacityByteArrayOutputStream[elementSizeInBytes];

  // Round-up the capacity hint.
  final int capacityPerStream = (pageSize + this.numStreams - 1) / this.numStreams;
  final int initialCapacityPerStream = (initialCapacity + this.numStreams - 1) / this.numStreams;
  for (int i = 0; i < this.numStreams; ++i) {
    this.byteStreams[i] = new CapacityByteArrayOutputStream(
            initialCapacityPerStream, capacityPerStream, allocator);
  }
}
 

private HadoopReadOptions(boolean useSignedStringMinMax,
                          boolean useStatsFilter,
                          boolean useDictionaryFilter,
                          boolean useRecordFilter,
                          boolean useColumnIndexFilter,
                          boolean usePageChecksumVerification,
                          boolean useBloomFilter,
                          FilterCompat.Filter recordFilter,
                          MetadataFilter metadataFilter,
                          CompressionCodecFactory codecFactory,
                          ByteBufferAllocator allocator,
                          int maxAllocationSize,
                          Map<String, String> properties,
                          Configuration conf) {
  super(
      useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
      usePageChecksumVerification, useBloomFilter, recordFilter, metadataFilter, codecFactory, allocator,
      maxAllocationSize, properties
  );
  this.conf = conf;
}
 

private ColumnChunkPageWriter(ColumnDescriptor path,
                              BytesCompressor compressor,
                              int initialSlabSize,
                              int maxCapacityHint,
                              ByteBufferAllocator allocator) {
  this.path = path;
  this.compressor = compressor;
  this.buf = new CapacityByteArrayOutputStream(initialSlabSize, maxCapacityHint, allocator);
  this.totalStatistics = Statistics.createStats(this.path.getPrimitiveType());
}
 

public ParquetColumnChunkPageWriteStore(BytesCompressor compressor,
                                        MessageType schema,
                                        int initialSlabSize,
                                        int maxCapacityHint,
                                        ByteBufferAllocator allocator) {
  this.schema = schema;
  for (ColumnDescriptor path : schema.getColumns()) {
    writers.put(path,  new ColumnChunkPageWriter(path, compressor, initialSlabSize, maxCapacityHint, allocator));
  }
}
 

/**
 * See docs on CodecFactory#createDirectCodecFactory which is how this class is
 * exposed publicly and is just a pass-through factory method for this constructor
 * to hide the rest of this class from public access.
 *
 * @throws NullPointerException if allocator is {@code null}
 */
DirectCodecFactory(Configuration config, ByteBufferAllocator allocator, int pageSize) {
  super(config, pageSize);

  this.allocator = Objects.requireNonNull(allocator, "allocator cannot be null");
  Preconditions.checkState(allocator.isDirect(),
      "A %s requires a direct buffer allocator be provided.",
      getClass().getSimpleName());
}
 

public RunLengthBitPackingHybridEncoder(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  LOG.debug("Encoding: RunLengthBitPackingHybridEncoder with "
    + "bithWidth: {} initialCapacity {}", bitWidth, initialCapacity);

  Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");

  this.bitWidth = bitWidth;
  this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
  this.packBuffer = new byte[bitWidth];
  this.bufferedValues = new int[8];
  this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
  reset(false);
}
 

private ColumnChunkPageWriter(ColumnDescriptor path,
                              BytesCompressor compressor,
                              ByteBufferAllocator allocator,
                              int columnIndexTruncateLength,
                              boolean pageWriteChecksumEnabled) {
  this.path = path;
  this.compressor = compressor;
  this.allocator = allocator;
  this.buf = new ConcatenatingByteArrayCollector();
  this.columnIndexBuilder = ColumnIndexBuilder.getBuilder(path.getPrimitiveType(), columnIndexTruncateLength);
  this.offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
  this.pageWriteChecksumEnabled = pageWriteChecksumEnabled;
  this.crc = pageWriteChecksumEnabled ? new CRC32() : null;
}
 

public DeltaLengthByteArrayValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) {
  arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator);
  out = new LittleEndianDataOutputStream(arrayOut);
  lengthWriter = new DeltaBinaryPackingValuesWriterForInteger(
      DeltaBinaryPackingValuesWriter.DEFAULT_NUM_BLOCK_VALUES,
      DeltaBinaryPackingValuesWriter.DEFAULT_NUM_MINIBLOCKS,
      initialSize, pageSize, allocator);
}
 

public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator,
    int columnIndexTruncateLength, boolean pageWriteChecksumEnabled) {
  this.schema = schema;
  for (ColumnDescriptor path : schema.getColumns()) {
    writers.put(path, new ColumnChunkPageWriter(path, compressor, allocator, columnIndexTruncateLength, pageWriteChecksumEnabled));
  }
}
 

ParquetReadOptions(boolean useSignedStringMinMax,
                   boolean useStatsFilter,
                   boolean useDictionaryFilter,
                   boolean useRecordFilter,
                   boolean useColumnIndexFilter,
                   boolean usePageChecksumVerification,
                   boolean useBloomFilter,
                   FilterCompat.Filter recordFilter,
                   ParquetMetadataConverter.MetadataFilter metadataFilter,
                   CompressionCodecFactory codecFactory,
                   ByteBufferAllocator allocator,
                   int maxAllocationSize,
                   Map<String, String> properties) {
  this.useSignedStringMinMax = useSignedStringMinMax;
  this.useStatsFilter = useStatsFilter;
  this.useDictionaryFilter = useDictionaryFilter;
  this.useRecordFilter = useRecordFilter;
  this.useColumnIndexFilter = useColumnIndexFilter;
  this.usePageChecksumVerification = usePageChecksumVerification;
  this.useBloomFilter = useBloomFilter;
  this.recordFilter = recordFilter;
  this.metadataFilter = metadataFilter;
  this.codecFactory = codecFactory;
  this.allocator = allocator;
  this.maxAllocationSize = maxAllocationSize;
  this.properties = Collections.unmodifiableMap(properties);
}
 

public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator,
                                 int columnIndexTruncateLength) {
  this(compressor, schema, allocator, columnIndexTruncateLength,
    ParquetProperties.DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED);
}
 

public ByteBufferAllocator getAllocator() {
  return allocator;
}
 

public static CompressionCodecFactory newDirectFactory(Configuration conf, ByteBufferAllocator allocator, int sizeHint) {
  return CodecFactory.createDirectCodecFactory(conf, allocator, sizeHint);
}
 

public Builder withAllocator(ByteBufferAllocator allocator) {
  this.allocator = allocator;
  return this;
}
 

public Builder withAllocator(ByteBufferAllocator allocator) {
  this.allocator = Objects.requireNonNull(allocator, "ByteBufferAllocator cannot be null");
  return this;
}
 

public ByteBufferAllocator getAllocator() {
  return allocator;
}
 

public FixedLenByteArrayPlainValuesWriter(int length, int initialSize, int pageSize, ByteBufferAllocator allocator) {
  this.length = length;
  this.allocator = allocator;
  this.arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, this.allocator);
  this.out = new LittleEndianDataOutputStream(arrayOut);
}
 

public PlainValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) {
  arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator);
  out = new LittleEndianDataOutputStream(arrayOut);
}
 

public FloatByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  super(Float.BYTES, initialCapacity, pageSize, allocator);
}
 

public DeltaByteArrayWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  this.prefixLengthWriter = 
      new DeltaBinaryPackingValuesWriterForInteger(128, 4, initialCapacity, pageSize, allocator);
  this.suffixWriter = new DeltaLengthByteArrayValuesWriter(initialCapacity, pageSize, allocator);
  this.previous = new byte[0];
}
 

public DeltaBinaryPackingValuesWriterForLong(
    int slabSize, int pageSize, ByteBufferAllocator allocator) {
  this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, slabSize, pageSize, allocator);
}
 

public PlainBinaryDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  binaryDictionaryContent.defaultReturnValue(-1);
}
 

public PlainFixedLenArrayDictionaryValuesWriter(int maxDictionaryByteSize, int length, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  this.length = length;
}
 

public PlainLongDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  longDictionaryContent.defaultReturnValue(-1);
}
 

public PlainDoubleDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  doubleDictionaryContent.defaultReturnValue(-1);
}
 

public PlainIntegerDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  intDictionaryContent.defaultReturnValue(-1);
}
 

public PlainFloatDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  floatDictionaryContent.defaultReturnValue(-1);
}
 

public RunLengthBitPackingHybridValuesWriter(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  this(new RunLengthBitPackingHybridEncoder(bitWidth, initialCapacity, pageSize, allocator));
}
 

public DoubleByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  super(Double.BYTES, initialCapacity, pageSize, allocator);
}
 

public DeltaBinaryPackingValuesWriterForLong(int blockSizeInValues, int miniBlockNum, 
    int slabSize, int pageSize, ByteBufferAllocator allocator) {
  super(blockSizeInValues, miniBlockNum, slabSize, pageSize, allocator);
  deltaBlockBuffer = new long[config.blockSizeInValues];
  miniBlockByteBuffer = new byte[config.miniBlockSizeInValues * MAX_BITWIDTH];
}