org.apache.parquet.bytes.ByteBufferAllocator Java Examples
The following examples show how to use
org.apache.parquet.bytes.ByteBufferAllocator.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ByteStreamSplitValuesWriter.java From parquet-mr with Apache License 2.0 | 6 votes |
public ByteStreamSplitValuesWriter(int elementSizeInBytes, int initialCapacity, int pageSize, ByteBufferAllocator allocator) { if (elementSizeInBytes <= 0) { throw new ParquetEncodingException(String.format("Element byte size is invalid: %d", elementSizeInBytes)); } this.numStreams = elementSizeInBytes; this.elementSizeInBytes = elementSizeInBytes; this.byteStreams = new CapacityByteArrayOutputStream[elementSizeInBytes]; // Round-up the capacity hint. final int capacityPerStream = (pageSize + this.numStreams - 1) / this.numStreams; final int initialCapacityPerStream = (initialCapacity + this.numStreams - 1) / this.numStreams; for (int i = 0; i < this.numStreams; ++i) { this.byteStreams[i] = new CapacityByteArrayOutputStream( initialCapacityPerStream, capacityPerStream, allocator); } }
Example #2
Source File: HadoopReadOptions.java From parquet-mr with Apache License 2.0 | 6 votes |
private HadoopReadOptions(boolean useSignedStringMinMax, boolean useStatsFilter, boolean useDictionaryFilter, boolean useRecordFilter, boolean useColumnIndexFilter, boolean usePageChecksumVerification, boolean useBloomFilter, FilterCompat.Filter recordFilter, MetadataFilter metadataFilter, CompressionCodecFactory codecFactory, ByteBufferAllocator allocator, int maxAllocationSize, Map<String, String> properties, Configuration conf) { super( useSignedStringMinMax, useStatsFilter, useDictionaryFilter, useRecordFilter, useColumnIndexFilter, usePageChecksumVerification, useBloomFilter, recordFilter, metadataFilter, codecFactory, allocator, maxAllocationSize, properties ); this.conf = conf; }
Example #3
Source File: ParquetColumnChunkPageWriteStore.java From Bats with Apache License 2.0 | 5 votes |
private ColumnChunkPageWriter(ColumnDescriptor path, BytesCompressor compressor, int initialSlabSize, int maxCapacityHint, ByteBufferAllocator allocator) { this.path = path; this.compressor = compressor; this.buf = new CapacityByteArrayOutputStream(initialSlabSize, maxCapacityHint, allocator); this.totalStatistics = Statistics.createStats(this.path.getPrimitiveType()); }
Example #4
Source File: ParquetColumnChunkPageWriteStore.java From Bats with Apache License 2.0 | 5 votes |
public ParquetColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, int initialSlabSize, int maxCapacityHint, ByteBufferAllocator allocator) { this.schema = schema; for (ColumnDescriptor path : schema.getColumns()) { writers.put(path, new ColumnChunkPageWriter(path, compressor, initialSlabSize, maxCapacityHint, allocator)); } }
Example #5
Source File: DirectCodecFactory.java From parquet-mr with Apache License 2.0 | 5 votes |
/** * See docs on CodecFactory#createDirectCodecFactory which is how this class is * exposed publicly and is just a pass-through factory method for this constructor * to hide the rest of this class from public access. * * @throws NullPointerException if allocator is {@code null} */ DirectCodecFactory(Configuration config, ByteBufferAllocator allocator, int pageSize) { super(config, pageSize); this.allocator = Objects.requireNonNull(allocator, "allocator cannot be null"); Preconditions.checkState(allocator.isDirect(), "A %s requires a direct buffer allocator be provided.", getClass().getSimpleName()); }
Example #6
Source File: RunLengthBitPackingHybridEncoder.java From parquet-mr with Apache License 2.0 | 5 votes |
public RunLengthBitPackingHybridEncoder(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) { LOG.debug("Encoding: RunLengthBitPackingHybridEncoder with " + "bithWidth: {} initialCapacity {}", bitWidth, initialCapacity); Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32"); this.bitWidth = bitWidth; this.baos = new CapacityByteArrayOutputStream(initialCapacity, pageSize, allocator); this.packBuffer = new byte[bitWidth]; this.bufferedValues = new int[8]; this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth); reset(false); }
Example #7
Source File: ColumnChunkPageWriteStore.java From parquet-mr with Apache License 2.0 | 5 votes |
private ColumnChunkPageWriter(ColumnDescriptor path, BytesCompressor compressor, ByteBufferAllocator allocator, int columnIndexTruncateLength, boolean pageWriteChecksumEnabled) { this.path = path; this.compressor = compressor; this.allocator = allocator; this.buf = new ConcatenatingByteArrayCollector(); this.columnIndexBuilder = ColumnIndexBuilder.getBuilder(path.getPrimitiveType(), columnIndexTruncateLength); this.offsetIndexBuilder = OffsetIndexBuilder.getBuilder(); this.pageWriteChecksumEnabled = pageWriteChecksumEnabled; this.crc = pageWriteChecksumEnabled ? new CRC32() : null; }
Example #8
Source File: DeltaLengthByteArrayValuesWriter.java From parquet-mr with Apache License 2.0 | 5 votes |
public DeltaLengthByteArrayValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) { arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator); out = new LittleEndianDataOutputStream(arrayOut); lengthWriter = new DeltaBinaryPackingValuesWriterForInteger( DeltaBinaryPackingValuesWriter.DEFAULT_NUM_BLOCK_VALUES, DeltaBinaryPackingValuesWriter.DEFAULT_NUM_MINIBLOCKS, initialSize, pageSize, allocator); }
Example #9
Source File: ColumnChunkPageWriteStore.java From parquet-mr with Apache License 2.0 | 5 votes |
public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator, int columnIndexTruncateLength, boolean pageWriteChecksumEnabled) { this.schema = schema; for (ColumnDescriptor path : schema.getColumns()) { writers.put(path, new ColumnChunkPageWriter(path, compressor, allocator, columnIndexTruncateLength, pageWriteChecksumEnabled)); } }
Example #10
Source File: ParquetReadOptions.java From parquet-mr with Apache License 2.0 | 5 votes |
ParquetReadOptions(boolean useSignedStringMinMax, boolean useStatsFilter, boolean useDictionaryFilter, boolean useRecordFilter, boolean useColumnIndexFilter, boolean usePageChecksumVerification, boolean useBloomFilter, FilterCompat.Filter recordFilter, ParquetMetadataConverter.MetadataFilter metadataFilter, CompressionCodecFactory codecFactory, ByteBufferAllocator allocator, int maxAllocationSize, Map<String, String> properties) { this.useSignedStringMinMax = useSignedStringMinMax; this.useStatsFilter = useStatsFilter; this.useDictionaryFilter = useDictionaryFilter; this.useRecordFilter = useRecordFilter; this.useColumnIndexFilter = useColumnIndexFilter; this.usePageChecksumVerification = usePageChecksumVerification; this.useBloomFilter = useBloomFilter; this.recordFilter = recordFilter; this.metadataFilter = metadataFilter; this.codecFactory = codecFactory; this.allocator = allocator; this.maxAllocationSize = maxAllocationSize; this.properties = Collections.unmodifiableMap(properties); }
Example #11
Source File: ColumnChunkPageWriteStore.java From parquet-mr with Apache License 2.0 | 4 votes |
public ColumnChunkPageWriteStore(BytesCompressor compressor, MessageType schema, ByteBufferAllocator allocator, int columnIndexTruncateLength) { this(compressor, schema, allocator, columnIndexTruncateLength, ParquetProperties.DEFAULT_PAGE_WRITE_CHECKSUM_ENABLED); }
Example #12
Source File: ParquetReadOptions.java From parquet-mr with Apache License 2.0 | 4 votes |
public ByteBufferAllocator getAllocator() { return allocator; }
Example #13
Source File: HadoopCodecs.java From parquet-mr with Apache License 2.0 | 4 votes |
public static CompressionCodecFactory newDirectFactory(Configuration conf, ByteBufferAllocator allocator, int sizeHint) { return CodecFactory.createDirectCodecFactory(conf, allocator, sizeHint); }
Example #14
Source File: ParquetReadOptions.java From parquet-mr with Apache License 2.0 | 4 votes |
public Builder withAllocator(ByteBufferAllocator allocator) { this.allocator = allocator; return this; }
Example #15
Source File: ParquetProperties.java From parquet-mr with Apache License 2.0 | 4 votes |
public Builder withAllocator(ByteBufferAllocator allocator) { this.allocator = Objects.requireNonNull(allocator, "ByteBufferAllocator cannot be null"); return this; }
Example #16
Source File: ParquetProperties.java From parquet-mr with Apache License 2.0 | 4 votes |
public ByteBufferAllocator getAllocator() { return allocator; }
Example #17
Source File: FixedLenByteArrayPlainValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public FixedLenByteArrayPlainValuesWriter(int length, int initialSize, int pageSize, ByteBufferAllocator allocator) { this.length = length; this.allocator = allocator; this.arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, this.allocator); this.out = new LittleEndianDataOutputStream(arrayOut); }
Example #18
Source File: PlainValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainValuesWriter(int initialSize, int pageSize, ByteBufferAllocator allocator) { arrayOut = new CapacityByteArrayOutputStream(initialSize, pageSize, allocator); out = new LittleEndianDataOutputStream(arrayOut); }
Example #19
Source File: ByteStreamSplitValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public FloatByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { super(Float.BYTES, initialCapacity, pageSize, allocator); }
Example #20
Source File: DeltaByteArrayWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public DeltaByteArrayWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { this.prefixLengthWriter = new DeltaBinaryPackingValuesWriterForInteger(128, 4, initialCapacity, pageSize, allocator); this.suffixWriter = new DeltaLengthByteArrayValuesWriter(initialCapacity, pageSize, allocator); this.previous = new byte[0]; }
Example #21
Source File: DeltaBinaryPackingValuesWriterForLong.java From parquet-mr with Apache License 2.0 | 4 votes |
public DeltaBinaryPackingValuesWriterForLong( int slabSize, int pageSize, ByteBufferAllocator allocator) { this(DEFAULT_NUM_BLOCK_VALUES, DEFAULT_NUM_MINIBLOCKS, slabSize, pageSize, allocator); }
Example #22
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainBinaryDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); binaryDictionaryContent.defaultReturnValue(-1); }
Example #23
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainFixedLenArrayDictionaryValuesWriter(int maxDictionaryByteSize, int length, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); this.length = length; }
Example #24
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainLongDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); longDictionaryContent.defaultReturnValue(-1); }
Example #25
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainDoubleDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); doubleDictionaryContent.defaultReturnValue(-1); }
Example #26
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainIntegerDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); intDictionaryContent.defaultReturnValue(-1); }
Example #27
Source File: DictionaryValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public PlainFloatDictionaryValuesWriter(int maxDictionaryByteSize, Encoding encodingForDataPage, Encoding encodingForDictionaryPage, ByteBufferAllocator allocator) { super(maxDictionaryByteSize, encodingForDataPage, encodingForDictionaryPage, allocator); floatDictionaryContent.defaultReturnValue(-1); }
Example #28
Source File: RunLengthBitPackingHybridValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public RunLengthBitPackingHybridValuesWriter(int bitWidth, int initialCapacity, int pageSize, ByteBufferAllocator allocator) { this(new RunLengthBitPackingHybridEncoder(bitWidth, initialCapacity, pageSize, allocator)); }
Example #29
Source File: ByteStreamSplitValuesWriter.java From parquet-mr with Apache License 2.0 | 4 votes |
public DoubleByteStreamSplitValuesWriter(int initialCapacity, int pageSize, ByteBufferAllocator allocator) { super(Double.BYTES, initialCapacity, pageSize, allocator); }
Example #30
Source File: DeltaBinaryPackingValuesWriterForLong.java From parquet-mr with Apache License 2.0 | 4 votes |
public DeltaBinaryPackingValuesWriterForLong(int blockSizeInValues, int miniBlockNum, int slabSize, int pageSize, ByteBufferAllocator allocator) { super(blockSizeInValues, miniBlockNum, slabSize, pageSize, allocator); deltaBlockBuffer = new long[config.blockSizeInValues]; miniBlockByteBuffer = new byte[config.miniBlockSizeInValues * MAX_BITWIDTH]; }