Java Code Examples for org.apache.parquet.bytes.BytesUtils#getWidthFromMaxInt()

The following examples show how to use org.apache.parquet.bytes.BytesUtils#getWidthFromMaxInt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractColumnReader.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void readPageV1(DataPageV1 page) throws IOException {
	this.pageValueCount = page.getValueCount();
	ValuesReader rlReader = page.getRlEncoding().getValuesReader(descriptor, REPETITION_LEVEL);

	// Initialize the decoders.
	if (page.getDlEncoding() != Encoding.RLE && descriptor.getMaxDefinitionLevel() != 0) {
		throw new UnsupportedOperationException("Unsupported encoding: " + page.getDlEncoding());
	}
	int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
	this.runLenDecoder = new RunLengthDecoder(bitWidth);
	try {
		BytesInput bytes = page.getBytes();
		ByteBufferInputStream in = bytes.toInputStream();
		rlReader.initFromPage(pageValueCount, in);
		this.runLenDecoder.initFromStream(pageValueCount, in);
		prepareNewPage(page.getValueEncoding(), in);
	} catch (IOException e) {
		throw new IOException("could not read page " + page + " in col " + descriptor, e);
	}
}
Example 2
Source File: PrimitiveColumnReader.java    From presto with Apache License 2.0 5 votes vote down vote up 
private LevelReader buildLevelRLEReader(int maxLevel, Slice slice)
{
    if (maxLevel == 0) {
        return new LevelNullReader();
    }
    return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), slice.getInput()));
}
Example 3
Source File: BasePageIterator.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
  try {
    if (maxLevel == 0) {
      return new PageIterator.NullIntIterator();
    }
    return new RLEIntIterator(
        new RunLengthBitPackingHybridDecoder(
            BytesUtils.getWidthFromMaxInt(maxLevel),
            bytes.toInputStream()));
  } catch (IOException e) {
    throw new ParquetDecodingException("could not read levels in page for col " + desc, e);
  }
}
Example 4
Source File: PageIterator.java    From iceberg with Apache License 2.0 5 votes vote down vote up 
private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
  try {
    if (maxLevel == 0) {
      return new NullIntIterator();
    }
    return new RLEIntIterator(
        new RunLengthBitPackingHybridDecoder(
            BytesUtils.getWidthFromMaxInt(maxLevel),
            bytes.toInputStream()));
  } catch (IOException e) {
    throw new ParquetDecodingException("could not read levels in page for col " + desc, e);
  }
}
Example 5
Source File: AbstractColumnReader.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readPageV2(DataPageV2 page) throws IOException {
	this.pageValueCount = page.getValueCount();

	int bitWidth = BytesUtils.getWidthFromMaxInt(descriptor.getMaxDefinitionLevel());
	// do not read the length from the stream. v2 pages handle dividing the page bytes.
	this.runLenDecoder = new RunLengthDecoder(bitWidth, false);
	this.runLenDecoder.initFromStream(
			this.pageValueCount, page.getDefinitionLevels().toInputStream());
	try {
		prepareNewPage(page.getDataEncoding(), page.getData().toInputStream());
	} catch (IOException e) {
		throw new IOException("could not read page " + page + " in col " + descriptor, e);
	}
}
Example 6
Source File: DictionaryValuesWriter.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
@Override
public BytesInput getBytes() {
  int maxDicId = getDictionarySize() - 1;
  LOG.debug("max dic id {}", maxDicId);
  int bitWidth = BytesUtils.getWidthFromMaxInt(maxDicId);
  int initialSlabSize =
      CapacityByteArrayOutputStream.initialSlabSizeHeuristic(MIN_INITIAL_SLAB_SIZE, maxDictionaryByteSize, 10);

  RunLengthBitPackingHybridEncoder encoder =
      new RunLengthBitPackingHybridEncoder(bitWidth, initialSlabSize, maxDictionaryByteSize, this.allocator);
  encoders.add(encoder);
  IntIterator iterator = encodedValues.iterator();
  try {
    while (iterator.hasNext()) {
      encoder.writeInt(iterator.next());
    }
    // encodes the bit width
    byte[] bytesHeader = new byte[] { (byte) bitWidth };
    BytesInput rleEncodedBytes = encoder.toBytes();
    LOG.debug("rle encoded bytes {}", rleEncodedBytes.size());
    BytesInput bytes = concat(BytesInput.from(bytesHeader), rleEncodedBytes);
    // remember size of dictionary when we last wrote a page
    lastUsedDictionarySize = getDictionarySize();
    lastUsedDictionaryByteSize = dictionaryByteSize;
    return bytes;
  } catch (IOException e) {
    throw new ParquetEncodingException("could not encode the values", e);
  }
}
Example 7
Source File: ColumnReaderBase.java    From parquet-mr with Apache License 2.0 5 votes vote down vote up 
private IntIterator newRLEIterator(int maxLevel, BytesInput bytes) {
  try {
    if (maxLevel == 0) {
      return new NullIntIterator();
    }
    return new RLEIntIterator(
        new RunLengthBitPackingHybridDecoder(
            BytesUtils.getWidthFromMaxInt(maxLevel),
            bytes.toInputStream()));
  } catch (IOException e) {
    throw new ParquetDecodingException("could not read levels in page for col " + path, e);
  }
}
Example 8
Source File: VectorizedPageIterator.java    From iceberg with Apache License 2.0 4 votes vote down vote up 
private VectorizedParquetDefinitionLevelReader newVectorizedDefinitionLevelReader(ColumnDescriptor desc) {
  int bitwidth = BytesUtils.getWidthFromMaxInt(desc.getMaxDefinitionLevel());
  return new VectorizedParquetDefinitionLevelReader(bitwidth, desc.getMaxDefinitionLevel(), setArrowValidityVector);
}
Example 9
Source File: ByteBitPackingValuesReader.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ByteBitPackingValuesReader(int bound, Packer packer) {
  this.bitWidth = BytesUtils.getWidthFromMaxInt(bound);
  this.packer = packer.newBytePacker(bitWidth);
}
Example 10
Source File: ByteBitPackingValuesWriter.java    From parquet-mr with Apache License 2.0 4 votes vote down vote up 
public ByteBitPackingValuesWriter(int bound, Packer packer) {
  this.packer = packer;
  this.bitWidth = BytesUtils.getWidthFromMaxInt(bound);
  this.encoder = new ByteBasedBitPackingEncoder(bitWidth, packer);
}