org.apache.parquet.bytes.ByteBufferAllocator Java Examples

The following examples show how to use org.apache.parquet.bytes.ByteBufferAllocator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ByteStreamSplitValuesWriter.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
public ByteStreamSplitValuesWriter(int elementSizeInBytes, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  if (elementSizeInBytes <= 0) {
    throw new ParquetEncodingException(String.format("Element byte size is invalid: %d", elementSizeInBytes));
  }
  this.numStreams = elementSizeInBytes;
  this.elementSizeInBytes = elementSizeInBytes;
  this.byteStreams = new CapacityByteArrayOutputStream[elementSizeInBytes];

  // Round-up the capacity hint.
  final int capacityPerStream = (pageSize + this.numStreams - 1) / this.numStreams;
  final int initialCapacityPerStream = (initialCapacity + this.numStreams - 1) / this.numStreams;
  for (int i = 0; i < this.numStreams; ++i) {
    this.byteStreams[i] = new CapacityByteArrayOutputStream(
            initialCapacityPerStream, capacityPerStream, allocator);
  }
}
 
Example #2
Source File: HadoopReadOptions.java    From parquet-mr with Apache License 2.0 6 votes vote down vote up
private HadoopReadOptions(boolean useSignedStringMinMax,
                          boolean useStatsFilter,
                          boolean useDictionaryFilter,
                          boolean useRecordFilter,
                          boolean useColumnIndexFilter,
                          boolean usePageChecksumVerification,
                          boolean useBloomFilter,
                          FilterCompat.Filter recordFilter,
                          MetadataFilter metadataFilter,
                          CompressionCodecFactory codecFactory,
                          ByteBufferAllocator allocator,
                          int maxAllocationSize,
                          Map<String, String> properties,
                          Configuration conf) {
  super(
      useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter,
      usePageChecksumVerification, useBloomFilter, recordFilter, metadataFilter, codecFactory, allocator,
      maxAllocationSize, properties
  );
  this.conf = conf;
}
 
Example #3
Source File: ParquetColumnChunkPageWriteStore.java    From Bats with Apache License 2.0 5 votes vote down vote up
private ColumnChunkPageWriter(ColumnDescriptor path,
                              BytesCompressor compressor,
                              int initialSlabSize,
                              int maxCapacityHint,
                              ByteBufferAllocator allocator) {
  this.path = path;
  this.compressor = compressor;
  this.buf = new CapacityByteArrayOutputStream(initialSlabSize, maxCapacityHint, allocator);
  this.totalStatistics = Statistics.createStats(this.path.getPrimitiveType());
}
 
Example #4
Source File: ParquetColumnChunkPageWriteStore.java    From Bats with Apache License 2.0 5 votes vote down vote up
public ParquetColumnChunkPageWriteStore(BytesCompressor compressor,
                                        MessageType schema,
                                        int initialSlabSize,
                                        int maxCapacityHint,
                                        ByteBufferAllocator allocator) {
  this.schema = schema;
  for (ColumnDescriptor path : schema.getColumns()) {
    writers.put(path,  new ColumnChunkPageWriter(path, compressor, initialSlabSize, maxCapacityHint, allocator));
  }
}
 
Example #5
Source File: DirectCodecFactory.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
/**
 * See docs on CodecFactory#createDirectCodecFactory which is how this class is
 * exposed publicly and is just a pass-through factory method for this constructor
 * to hide the rest of this class from public access.
 *
 * @throws NullPointerException if allocator is {@code null}
 */
DirectCodecFactory(Configuration config, ByteBufferAllocator allocator, int pageSize) {
  super(config, pageSize);

  this.allocator = Objects.requireNonNull(allocator, "allocator cannot be null");
  Preconditions.checkState(allocator.isDirect(),
      "A %s requires a direct buffer allocator be provided.",
      getClass().getSimpleName());
}
 
Example #6
Source File: RunLengthBitPackingHybridEncoder.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
public RunLengthBitPackingHybridEncoder(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  LOG.debug("Encoding: RunLengthBitPackingHybridEncoder with "
    + "bithWidth: {} initialCapacity {}", bitWidth, initialCapacity);

  Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");

  this.bitWidth = bitWidth;
  this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator);
  this.packBuffer = new byte[bitWidth];
  this.bufferedValues = new int[8];
  this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
  reset(false);
}
 
Example #7
Source File: ColumnChunkPageWriteStore.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
private ColumnChunkPageWriter(ColumnDescriptor path,
                              BytesCompressor compressor,
                              ByteBufferAllocator allocator,
                              int columnIndexTruncateLength,
                              boolean pageWriteChecksumEnabled) {
  this.path = path;
  this.compressor = compressor;
  this.allocator = allocator;
  this.buf = new ConcatenatingByteArrayCollector();
  this.columnIndexBuilder = ColumnIndexBuilder.getBuilder(path.getPrimitiveType(), columnIndexTruncateLength);
  this.offsetIndexBuilder = OffsetIndexBuilder.getBuilder();
  this.pageWriteChecksumEnabled = pageWriteChecksumEnabled;
  this.crc = pageWriteChecksumEnabled ? new CRC32() : null;
}
 
Example #8
Source File: DeltaLengthByteArrayValuesWriter.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
public DeltaLengthByteArrayValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) {
  arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator);
  out = new LittleEndianDataOutputStream(arrayOut);
  lengthWriter = new DeltaBinaryPackingValuesWriterForInteger(
      DeltaBinaryPackingValuesWriter.DEFAULT_NUM_BLOCK_VALUES,
      DeltaBinaryPackingValuesWriter.DEFAULT_NUM_MINIBLOCKS,
      initialSize, pageSize, allocator);
}
 
Example #9
Source File: ColumnChunkPageWriteStore.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator,
    int columnIndexTruncateLength, boolean pageWriteChecksumEnabled) {
  this.schema = schema;
  for (ColumnDescriptor path : schema.getColumns()) {
    writers.put(path, new ColumnChunkPageWriter(path, compressor, allocator, columnIndexTruncateLength, pageWriteChecksumEnabled));
  }
}
 
Example #10
Source File: ParquetReadOptions.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up
ParquetReadOptions(boolean useSignedStringMinMax,
                   boolean useStatsFilter,
                   boolean useDictionaryFilter,
                   boolean useRecordFilter,
                   boolean useColumnIndexFilter,
                   boolean usePageChecksumVerification,
                   boolean useBloomFilter,
                   FilterCompat.Filter recordFilter,
                   ParquetMetadataConverter.MetadataFilter metadataFilter,
                   CompressionCodecFactory codecFactory,
                   ByteBufferAllocator allocator,
                   int maxAllocationSize,
                   Map<String, String> properties) {
  this.useSignedStringMinMax = useSignedStringMinMax;
  this.useStatsFilter = useStatsFilter;
  this.useDictionaryFilter = useDictionaryFilter;
  this.useRecordFilter = useRecordFilter;
  this.useColumnIndexFilter = useColumnIndexFilter;
  this.usePageChecksumVerification = usePageChecksumVerification;
  this.useBloomFilter = useBloomFilter;
  this.recordFilter = recordFilter;
  this.metadataFilter = metadataFilter;
  this.codecFactory = codecFactory;
  this.allocator = allocator;
  this.maxAllocationSize = maxAllocationSize;
  this.properties = Collections.unmodifiableMap(properties);
}
 
Example #11
Source File: ColumnChunkPageWriteStore.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator,
                                 int columnIndexTruncateLength) {
  this(compressor, schema, allocator, columnIndexTruncateLength,
    ParquetProperties.DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED);
}
 
Example #12
Source File: ParquetReadOptions.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public ByteBufferAllocator getAllocator() {
  return allocator;
}
 
Example #13
Source File: HadoopCodecs.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public static CompressionCodecFactory newDirectFactory(Configuration conf, ByteBufferAllocator allocator, int sizeHint) {
  return CodecFactory.createDirectCodecFactory(conf, allocator, sizeHint);
}
 
Example #14
Source File: ParquetReadOptions.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public Builder withAllocator(ByteBufferAllocator allocator) {
  this.allocator = allocator;
  return this;
}
 
Example #15
Source File: ParquetProperties.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public Builder withAllocator(ByteBufferAllocator allocator) {
  this.allocator = Objects.requireNonNull(allocator, "ByteBufferAllocator cannot be null");
  return this;
}
 
Example #16
Source File: ParquetProperties.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public ByteBufferAllocator getAllocator() {
  return allocator;
}
 
Example #17
Source File: FixedLenByteArrayPlainValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public FixedLenByteArrayPlainValuesWriter(int length, int initialSize, int pageSize, ByteBufferAllocator allocator) {
  this.length = length;
  this.allocator = allocator;
  this.arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, this.allocator);
  this.out = new LittleEndianDataOutputStream(arrayOut);
}
 
Example #18
Source File: PlainValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) {
  arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator);
  out = new LittleEndianDataOutputStream(arrayOut);
}
 
Example #19
Source File: ByteStreamSplitValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public FloatByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  super(Float.BYTES, initialCapacity, pageSize, allocator);
}
 
Example #20
Source File: DeltaByteArrayWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public DeltaByteArrayWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  this.prefixLengthWriter = 
      new DeltaBinaryPackingValuesWriterForInteger(128, 4, initialCapacity, pageSize, allocator);
  this.suffixWriter = new DeltaLengthByteArrayValuesWriter(initialCapacity, pageSize, allocator);
  this.previous = new byte[0];
}
 
Example #21
Source File: DeltaBinaryPackingValuesWriterForLong.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public DeltaBinaryPackingValuesWriterForLong(
    int slabSize, int pageSize, ByteBufferAllocator allocator) {
  this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, slabSize, pageSize, allocator);
}
 
Example #22
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainBinaryDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  binaryDictionaryContent.defaultReturnValue(-1);
}
 
Example #23
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainFixedLenArrayDictionaryValuesWriter(int maxDictionaryByteSize, int length, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  this.length = length;
}
 
Example #24
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainLongDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  longDictionaryContent.defaultReturnValue(-1);
}
 
Example #25
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainDoubleDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  doubleDictionaryContent.defaultReturnValue(-1);
}
 
Example #26
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainIntegerDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  intDictionaryContent.defaultReturnValue(-1);
}
 
Example #27
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public PlainFloatDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) {
  super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator);
  floatDictionaryContent.defaultReturnValue(-1);
}
 
Example #28
Source File: RunLengthBitPackingHybridValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public RunLengthBitPackingHybridValuesWriter(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  this(new RunLengthBitPackingHybridEncoder(bitWidth, initialCapacity, pageSize, allocator));
}
 
Example #29
Source File: ByteStreamSplitValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public DoubleByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) {
  super(Double.BYTES, initialCapacity, pageSize, allocator);
}
 
Example #30
Source File: DeltaBinaryPackingValuesWriterForLong.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up
public DeltaBinaryPackingValuesWriterForLong(int blockSizeInValues, int miniBlockNum, 
    int slabSize, int pageSize, ByteBufferAllocator allocator) {
  super(blockSizeInValues, miniBlockNum, slabSize, pageSize, allocator);
  deltaBlockBuffer = new long[config.blockSizeInValues];
  miniBlockByteBuffer = new byte[config.miniBlockSizeInValues * MAX_BITWIDTH];
}