Java Code Examples for io.prestosql.spi.block.BlockBuilder#newBlockBuilderLike()

The following examples show how to use io.prestosql.spi.block.BlockBuilder#newBlockBuilderLike() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestByteArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new ByteArrayBlockBuilder(null, 0);

    BlockBuilder blockBuilder = new ByteArrayBlockBuilder(null, expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 2

Source File: TestInt128ArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0);

    BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 3

Source File: TestLongArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0);

    BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 4

Source File: TestIntArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new IntArrayBlockBuilder(null, 0);

    BlockBuilder blockBuilder = new IntArrayBlockBuilder(null, expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 5

Source File: TestArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    long[][] expectedValues = new long[ARRAY_SIZES.length][];
    Random rand = new Random(47);
    for (int i = 0; i < ARRAY_SIZES.length; i++) {
        expectedValues[i] = rand.longs(ARRAY_SIZES[i]).toArray();
    }
    BlockBuilder emptyBlockBuilder = new ArrayBlockBuilder(BIGINT, null, 0, 0);

    BlockBuilder blockBuilder = new ArrayBlockBuilder(BIGINT, null, 100, 100);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 6

Source File: TestShortArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new ShortArrayBlockBuilder(null, 0);

    BlockBuilder blockBuilder = new ShortArrayBlockBuilder(null, expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 7

Source File: TestInt96ArrayBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createTestValue(100);
    BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0);

    BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 8

Source File: TestVariableWidthBlock.java From presto with Apache License 2.0

6 votes

@Test
public void testLazyBlockBuilderInitialization()
{
    Slice[] expectedValues = createExpectedValues(100);
    BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0);

    BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());

    writeValues(expectedValues, blockBuilder);
    assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes());
    assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes());

    blockBuilder = blockBuilder.newBlockBuilderLike(null);
    assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes());
    assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes());
}

Example 9

Source File: TestOrcWriter.java From presto with Apache License 2.0

4 votes

@Test
public void testWriteOutputStreamsInOrder()
        throws IOException
{
    for (OrcWriteValidationMode validationMode : OrcWriteValidationMode.values()) {
        TempFile tempFile = new TempFile();

        List<String> columnNames = ImmutableList.of("test1", "test2", "test3", "test4", "test5");
        List<Type> types = ImmutableList.of(VARCHAR, VARCHAR, VARCHAR, VARCHAR, VARCHAR);

        OrcWriter writer = new OrcWriter(
                new OutputStreamOrcDataSink(new FileOutputStream(tempFile.getFile())),
                ImmutableList.of("test1", "test2", "test3", "test4", "test5"),
                types,
                OrcType.createRootOrcType(columnNames, types),
                NONE,
                new OrcWriterOptions()
                        .withStripeMinSize(DataSize.of(0, MEGABYTE))
                        .withStripeMaxSize(DataSize.of(32, MEGABYTE))
                        .withStripeMaxRowCount(ORC_STRIPE_SIZE)
                        .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE)
                        .withDictionaryMaxMemory(DataSize.of(32, MEGABYTE)),
                false,
                ImmutableMap.of(),
                HIVE_STORAGE_TIME_ZONE,
                true,
                validationMode,
                new OrcWriterStats());

        // write down some data with unsorted streams
        String[] data = new String[] {"a", "bbbbb", "ccc", "dd", "eeee"};
        Block[] blocks = new Block[data.length];
        int entries = 65536;
        BlockBuilder blockBuilder = VARCHAR.createBlockBuilder(null, entries);
        for (int i = 0; i < data.length; i++) {
            byte[] bytes = data[i].getBytes(UTF_8);
            for (int j = 0; j < entries; j++) {
                // force to write different data
                bytes[0] = (byte) ((bytes[0] + 1) % 128);
                blockBuilder.writeBytes(Slices.wrappedBuffer(bytes, 0, bytes.length), 0, bytes.length);
                blockBuilder.closeEntry();
            }
            blocks[i] = blockBuilder.build();
            blockBuilder = blockBuilder.newBlockBuilderLike(null);
        }

        writer.write(new Page(blocks));
        writer.close();

        // read the footer and verify the streams are ordered by size
        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), READER_OPTIONS);
        Footer footer = new OrcReader(orcDataSource, READER_OPTIONS).getFooter();

        for (StripeInformation stripe : footer.getStripes()) {
            // read the footer
            Slice tailBuffer = orcDataSource.readFully(stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength(), toIntExact(stripe.getFooterLength()));
            try (InputStream inputStream = new OrcInputStream(OrcChunkLoader.create(orcDataSource.getId(), tailBuffer, Optional.empty(), newSimpleAggregatedMemoryContext()))) {
                StripeFooter stripeFooter = new OrcMetadataReader().readStripeFooter(footer.getTypes(), inputStream);

                int size = 0;
                boolean dataStreamStarted = false;
                for (Stream stream : stripeFooter.getStreams()) {
                    if (isIndexStream(stream)) {
                        assertFalse(dataStreamStarted);
                        continue;
                    }
                    dataStreamStarted = true;
                    // verify sizes in order
                    assertGreaterThanOrEqual(stream.getLength(), size);
                    size = stream.getLength();
                }
            }
        }
    }
}

Example 10

Source File: TestStructColumnReader.java From presto with Apache License 2.0

4 votes

private void write(TempFile tempFile, Type writerType, List<String> data)
        throws IOException
{
    List<String> columnNames = ImmutableList.of(STRUCT_COL_NAME);
    List<Type> types = ImmutableList.of(writerType);
    OrcWriter writer = new OrcWriter(
            new OutputStreamOrcDataSink(new FileOutputStream(tempFile.getFile())),
            columnNames,
            types,
            OrcType.createRootOrcType(columnNames, types),
            NONE,
            new OrcWriterOptions()
                    .withStripeMinSize(DataSize.of(0, MEGABYTE))
                    .withStripeMaxSize(DataSize.of(32, MEGABYTE))
                    .withStripeMaxRowCount(ORC_STRIPE_SIZE)
                    .withRowGroupMaxRowCount(ORC_ROW_GROUP_SIZE)
                    .withDictionaryMaxMemory(DataSize.of(32, MEGABYTE)),
            false,
            ImmutableMap.of(),
            HIVE_STORAGE_TIME_ZONE,
            true,
            BOTH,
            new OrcWriterStats());

    // write down some data with unsorted streams
    Block[] fieldBlocks = new Block[data.size()];

    int entries = 10;
    boolean[] rowIsNull = new boolean[entries];
    Arrays.fill(rowIsNull, false);

    BlockBuilder blockBuilder = TEST_DATA_TYPE.createBlockBuilder(null, entries);
    for (int i = 0; i < data.size(); i++) {
        byte[] bytes = data.get(i).getBytes(UTF_8);
        for (int j = 0; j < entries; j++) {
            blockBuilder.writeBytes(Slices.wrappedBuffer(bytes), 0, bytes.length);
            blockBuilder.closeEntry();
        }
        fieldBlocks[i] = blockBuilder.build();
        blockBuilder = blockBuilder.newBlockBuilderLike(null);
    }
    Block rowBlock = RowBlock.fromFieldBlocks(rowIsNull.length, Optional.of(rowIsNull), fieldBlocks);
    writer.write(new Page(rowBlock));
    writer.close();
}