io.airlift.slice.Slice Java Examples

The following examples show how to use io.airlift.slice.Slice. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestColumnarArray.java    From presto with Apache License 2.0 7 votes vote down vote up
@Test
public void test()
{
    Slice[][] expectedValues = new Slice[ARRAY_SIZES.length][];
    for (int i = 0; i < ARRAY_SIZES.length; i++) {
        expectedValues[i] = new Slice[ARRAY_SIZES[i]];
        for (int j = 0; j < ARRAY_SIZES[i]; j++) {
            if (j % 3 != 1) {
                expectedValues[i][j] = Slices.utf8Slice(format("%d.%d", i, j));
            }
        }
    }
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    verifyBlock(blockBuilder, expectedValues);
    verifyBlock(blockBuilder.build(), expectedValues);

    Slice[][] expectedValuesWithNull = alternatingNullValues(expectedValues);
    BlockBuilder blockBuilderWithNull = createBlockBuilderWithValues(expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull, expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull.build(), expectedValuesWithNull);
}
 
Example #2
Source File: Chars.java    From presto with Apache License 2.0 7 votes vote down vote up
public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 
Example #3
Source File: DateTimeFunctions.java    From presto with Apache License 2.0 7 votes vote down vote up
private static DateTimeField getDateField(ISOChronology chronology, Slice unit)
{
    String unitString = unit.toStringUtf8().toLowerCase(ENGLISH);
    switch (unitString) {
        case "day":
            return chronology.dayOfMonth();
        case "week":
            return chronology.weekOfWeekyear();
        case "month":
            return chronology.monthOfYear();
        case "quarter":
            return QUARTER_OF_YEAR.getField(chronology);
        case "year":
            return chronology.year();
    }
    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "'" + unitString + "' is not a valid DATE field");
}
 
Example #4
Source File: TestOrcReaderPositions.java    From spliceengine with GNU Affero General Public License v3.0 7 votes vote down vote up
public void testReadUserMetadata()
        throws Exception
{
    try (TempFile tempFile = new TempFile()) {
        Map<String, String> metadata = ImmutableMap.of(
                "a", "ala",
                "b", "ma",
                "c", "kota");
        createFileWithOnlyUserMetadata(tempFile.getFile(), metadata);

        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        Footer footer = orcReader.getFooter();
        Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii);
        assertEquals(readMetadata, metadata);
    }
}
 
Example #5
Source File: BlockEncoding.java    From presto with Apache License 2.0 7 votes vote down vote up
@Override
public final Block decodeColumn(ColumnData columnData)
        throws RcFileCorruptionException
{
    int size = columnData.rowCount();

    Slice slice = columnData.getSlice();
    BlockBuilder builder = type.createBlockBuilder(null, size);
    for (int i = 0; i < size; i++) {
        int length = columnData.getLength(i);
        int offset = columnData.getOffset(i);
        if (!isNullSequence(slice, offset, length)) {
            decodeValueInto(1, builder, slice, offset, length);
        }
        else {
            builder.appendNull();
        }
    }

    return builder.build();
}
 
Example #6
Source File: Footer.java    From presto with Apache License 2.0 6 votes vote down vote up
public Footer(
        long numberOfRows,
        OptionalInt rowsInRowGroup,
        List<StripeInformation> stripes,
        ColumnMetadata<OrcType> types,
        Optional<ColumnMetadata<ColumnStatistics>> fileStats,
        Map<String, Slice> userMetadata)
{
    this.numberOfRows = numberOfRows;
    rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1"));
    this.rowsInRowGroup = rowsInRowGroup;
    this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null"));
    this.types = requireNonNull(types, "types is null");
    this.fileStats = requireNonNull(fileStats, "fileStats is null");
    requireNonNull(userMetadata, "userMetadata is null");
    this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slices::copyOf));
}
 
Example #7
Source File: DataPageV2.java    From presto with Apache License 2.0 6 votes vote down vote up
public DataPageV2(
        int rowCount,
        int nullCount,
        int valueCount,
        Slice repetitionLevels,
        Slice definitionLevels,
        ParquetEncoding dataEncoding,
        Slice slice,
        int uncompressedSize,
        Statistics<?> statistics,
        boolean isCompressed)
{
    super(uncompressedSize, valueCount);
    this.rowCount = rowCount;
    this.nullCount = nullCount;
    this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
    this.definitionLevels = requireNonNull(definitionLevels, "definitionLevels slice is null");
    this.dataEncoding = dataEncoding;
    this.slice = requireNonNull(slice, "slice is null");
    this.statistics = statistics;
    this.isCompressed = isCompressed;
}
 
Example #8
Source File: ColorFunctions.java    From presto with Apache License 2.0 6 votes vote down vote up
@ScalarFunction
@LiteralParameters({"x", "y"})
@Constraint(variable = "y", expression = "min(2147483647, x + 15)")
// Color formatting uses 15 characters. Note that if the ansiColorEscape function implementation
// changes, this value may be invalidated.
@SqlType("varchar(y)")
public static Slice render(@SqlType("varchar(x)") Slice value, @SqlType(ColorType.NAME) long color)
{
    StringBuilder builder = new StringBuilder(value.length());

    // color
    builder.append(ansiColorEscape(color))
            .append(value.toStringUtf8())
            .append(ANSI_RESET);

    return utf8Slice(builder.toString());
}
 
Example #9
Source File: DecimalColumnReader.java    From presto with Apache License 2.0 6 votes vote down vote up
private Block readLongNotNullBlock()
        throws IOException
{
    verifyNotNull(decimalStream);
    verifyNotNull(scaleStream);

    long[] data = new long[nextBatchSize * 2];
    decimalStream.nextLongDecimal(data, nextBatchSize);

    for (int offset = 0; offset < data.length; offset += 2) {
        long sourceScale = scaleStream.next();
        if (sourceScale != type.getScale()) {
            Slice decimal = Slices.wrappedLongArray(data[offset], data[offset + 1]);
            UnscaledDecimal128Arithmetic.rescale(decimal, (int) (type.getScale() - sourceScale), Slices.wrappedLongArray(data, offset, 2));
        }
    }
    return new Int128ArrayBlock(nextBatchSize, Optional.empty(), data);
}
 
Example #10
Source File: UnscaledDecimal128Arithmetic.java    From presto with Apache License 2.0 6 votes vote down vote up
public static void subtract(Slice left, Slice right, Slice result)
{
    if (isNegative(left) != isNegative(right)) {
        // only one is negative
        if (addUnsignedReturnOverflow(left, right, result, isNegative(left)) != 0) {
            throwOverflowException();
        }
    }
    else {
        int compare = compareAbsolute(left, right);
        if (compare > 0) {
            subtractUnsigned(left, right, result, isNegative(left) && isNegative(right));
        }
        else if (compare < 0) {
            subtractUnsigned(right, left, result, !(isNegative(left) && isNegative(right)));
        }
        else {
            setToZero(result);
        }
    }
}
 
Example #11
Source File: LikeFunctions.java    From presto with Apache License 2.0 6 votes vote down vote up
public static int patternConstantPrefixBytes(Slice pattern, Optional<Slice> escape)
{
    int escapeChar = getEscapeCharacter(escape)
            .map(c -> (int) c)
            .orElse(-1);

    boolean escaped = false;
    int position = 0;
    while (position < pattern.length()) {
        int currentChar = getCodePointAt(pattern, position);
        if (!escaped && (currentChar == escapeChar)) {
            escaped = true;
        }
        else if (escaped) {
            checkEscape(currentChar == '%' || currentChar == '_' || currentChar == escapeChar);
            escaped = false;
        }
        else if ((currentChar == '%') || (currentChar == '_')) {
            return position;
        }
        position += lengthOfCodePoint(currentChar);
    }
    checkEscape(!escaped);
    return position;
}
 
Example #12
Source File: UnscaledDecimal128Arithmetic.java    From presto with Apache License 2.0 6 votes vote down vote up
public static Slice pack(BigInteger unscaledValue, Slice result)
{
    pack(0, 0, false, result);
    byte[] bytes = unscaledValue.abs().toByteArray();

    if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH
            || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) {
        throwOverflowException();
    }

    // convert to little-endian order
    reverse(bytes);

    result.setBytes(0, bytes);
    if (unscaledValue.signum() < 0) {
        setNegative(result, true);
    }

    throwIfOverflows(result);

    return result;
}
 
Example #13
Source File: PrimitiveColumnReader.java    From presto with Apache License 2.0 5 votes vote down vote up
private LevelReader buildLevelRLEReader(int maxLevel, Slice slice)
{
    if (maxLevel == 0) {
        return new LevelNullReader();
    }
    return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), slice.getInput()));
}
 
Example #14
Source File: AtopPageSource.java    From presto with Apache License 2.0 5 votes vote down vote up
public AtopPageSource(Semaphore readerPermits, AtopFactory atopFactory, ConnectorSession session, Slice hostIp, AtopTable table, ZonedDateTime date, List<AtopColumn> columns, List<Type> types)
{
    this.readerPermits = requireNonNull(readerPermits, "readerPermits is null");
    this.atopFactory = requireNonNull(atopFactory, "atopFactory is null");
    this.session = requireNonNull(session, "session is null");
    this.hostIp = requireNonNull(hostIp, "hostIp is null");
    this.table = requireNonNull(table, "table is null");
    this.date = requireNonNull(date, "date is null");
    this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
    this.types = ImmutableList.copyOf(requireNonNull(types, "types is null"));
    checkArgument(columns.size() == types.size(), "columns (%s) does not match types (%s)", columns.size(), types.size());
    pageBuilder = new PageBuilder(types);
}
 
Example #15
Source File: TestInt128ArrayBlock.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyPositions()
{
    Slice[] expectedValues = alternatingNullValues(createTestValue(17));
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    assertBlockFilteredPositions(expectedValues, blockBuilder.build(), () -> blockBuilder.newBlockBuilderLike(null), 0, 2, 4, 6, 7, 9, 10, 16);
}
 
Example #16
Source File: TestUnscaledDecimal128Arithmetic.java    From presto with Apache License 2.0 5 votes vote down vote up
private static void assertDivideAllSigns(Slice dividend, int dividendRescaleFactor, Slice divisor, int divisorRescaleFactor)
{
    assertDivide(dividend, dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(dividend, dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
}
 
Example #17
Source File: MatchQueryFunction.java    From presto-connectors with Apache License 2.0 5 votes vote down vote up
@ScalarFunction("match_query")
@Description("es match_query")
@SqlType(StandardTypes.VARCHAR)
@SqlNullable
public static Slice matchQuery(
        @SqlType(StandardTypes.VARCHAR) Slice filter)
{
    if (filter == null) {
        return null;
    }
    String filterStr = filter.toStringUtf8();

    QueryBuilder builder = QueryBuilders.matchQuery(MATCH_COLUMN_SEP, filterStr);
    return Slices.utf8Slice(builder.toString());
}
 
Example #18
Source File: TestLongDecimalStream.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
protected DecimalInputStream createValueStream(Slice slice)
        throws OrcCorruptionException
{
    Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE);
    return new DecimalInputStream(OrcChunkLoader.create(ORC_DATA_SOURCE_ID, slice, orcDecompressor, newSimpleAggregatedMemoryContext()));
}
 
Example #19
Source File: InputReferenceCompiler.java    From presto with Apache License 2.0 5 votes vote down vote up
private InputReferenceNode(CallSiteBinder callSiteBinder, Scope scope, Type type, BytecodeExpression block, BytecodeExpression position)
{
    // Generate body based on block and position
    Variable wasNullVariable = scope.getVariable("wasNull");
    Class<?> callType = type.getJavaType();
    if (!callType.isPrimitive() && callType != Slice.class) {
        callType = Object.class;
    }

    IfStatement ifStatement = new IfStatement();
    ifStatement.condition(block.invoke("isNull", boolean.class, position));

    ifStatement.ifTrue()
            .putVariable(wasNullVariable, true)
            .pushJavaDefault(callType);

    String methodName = "get" + Primitives.wrap(callType).getSimpleName();
    BytecodeExpression value = constantType(callSiteBinder, type).invoke(methodName, callType, block, position);
    if (callType != type.getJavaType()) {
        value = value.cast(type.getJavaType());
    }
    ifStatement.ifFalse(value);

    this.body = ifStatement;
    this.block = block;
    this.position = position;
}
 
Example #20
Source File: BingTileFunctions.java    From presto with Apache License 2.0 5 votes vote down vote up
@Description("Creates a Bing tile from a QuadKey")
@ScalarFunction("bing_tile")
@SqlType(BingTileType.NAME)
public static long toBingTile(@SqlType(StandardTypes.VARCHAR) Slice quadKey)
{
    checkQuadKey(quadKey);
    return BingTile.fromQuadKey(quadKey.toStringUtf8()).encode();
}
 
Example #21
Source File: PagesSerde.java    From presto with Apache License 2.0 5 votes vote down vote up
public Page deserialize(SerializedPage serializedPage)
{
    checkArgument(serializedPage != null, "serializedPage is null");

    Slice slice = serializedPage.getSlice();

    if (serializedPage.isEncrypted()) {
        checkState(spillCipher.isPresent(), "Page is encrypted, but spill cipher is missing");

        byte[] decrypted = new byte[spillCipher.get().decryptedMaxLength(slice.length())];
        int decryptedSize = spillCipher.get().decrypt(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decrypted,
                0);

        slice = Slices.wrappedBuffer(decrypted, 0, decryptedSize);
    }

    if (serializedPage.isCompressed()) {
        checkState(decompressor.isPresent(), "Page is compressed, but decompressor is missing");

        int uncompressedSize = serializedPage.getUncompressedSizeInBytes();
        byte[] decompressed = new byte[uncompressedSize];
        checkState(decompressor.get().decompress(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decompressed,
                0,
                uncompressedSize) == uncompressedSize);

        slice = Slices.wrappedBuffer(decompressed);
    }

    return readRawPage(serializedPage.getPositionCount(), slice.getInput(), blockEncodingSerde);
}
 
Example #22
Source File: CharacterStringCasts.java    From presto with Apache License 2.0 5 votes vote down vote up
@ScalarOperator(OperatorType.CAST)
@SqlType("char(y)")
@LiteralParameters({"x", "y"})
public static Slice charToCharCast(@LiteralParameter("x") Long x, @LiteralParameter("y") Long y, @SqlType("char(x)") Slice slice)
{
    if (x > y) {
        return truncateToLength(slice, y.intValue());
    }
    else {
        return slice;
    }
}
 
Example #23
Source File: DecimalOperators.java    From presto with Apache License 2.0 5 votes vote down vote up
@LiteralParameters({"p", "s"})
@SqlType("decimal(p, s)")
public static Slice negate(@SqlType("decimal(p, s)") Slice arg)
{
    BigInteger argBigInteger = Decimals.decodeUnscaledValue(arg);
    return encodeUnscaledValue(argBigInteger.negate());
}
 
Example #24
Source File: InformationSchemaMetadata.java    From presto with Apache License 2.0 5 votes vote down vote up
private <T> Optional<Set<String>> filterString(TupleDomain<T> constraint, T column)
{
    if (constraint.isNone()) {
        return Optional.of(ImmutableSet.of());
    }

    Domain domain = constraint.getDomains().get().get(column);
    if (domain == null) {
        return Optional.empty();
    }

    if (domain.isSingleValue()) {
        return Optional.of(ImmutableSet.of(((Slice) domain.getSingleValue()).toStringUtf8()));
    }
    if (domain.getValues() instanceof EquatableValueSet) {
        Collection<Object> values = ((EquatableValueSet) domain.getValues()).getValues();
        return Optional.of(values.stream()
                .map(Slice.class::cast)
                .map(Slice::toStringUtf8)
                .collect(toImmutableSet()));
    }
    if (domain.getValues() instanceof SortedRangeSet) {
        ImmutableSet.Builder<String> result = ImmutableSet.builder();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            checkState(!range.isAll()); // Already checked
            if (!range.isSingleValue()) {
                return Optional.empty();
            }

            result.add(((Slice) range.getSingleValue()).toStringUtf8());
        }

        return Optional.of(result.build());
    }
    return Optional.empty();
}
 
Example #25
Source File: DecimalEncoding.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public void decodeValueInto(BlockBuilder builder, Slice slice, int offset, int length)
{
    if (isShortDecimal(type)) {
        type.writeLong(builder, parseLong(slice, offset));
    }
    else {
        type.writeSlice(builder, parseSlice(slice, offset));
    }
}
 
Example #26
Source File: TestArrayUnnester.java    From presto with Apache License 2.0 5 votes vote down vote up
@Test
public void testTrimmedBlocks()
{
    int[] unnestedLengths = {2, 1, 2, 3, 1};

    Slice[][][] elements = column(
            array(toSlices("0.0.0"), toSlices("0.1.0")),
            array(toSlices("1.0.0")),
            array(toSlices("2.0.0"), toSlices("2.1.0")),
            array(toSlices("3.0.0"), toSlices("3.1.0"), toSlices("3.2.0")),
            array(toSlices("4.0.0")));

    Slice[][] slices = getFieldElements(elements, 0);
    Block arrayBlock = createArrayBlock(slices);

    // Remove the first element from the arrayBlock for testing
    int startElement = 1;

    Slice[][] truncatedSlices = Arrays.copyOfRange(slices, startElement, slices.length - startElement + 1);
    int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, slices.length - startElement + 1);
    Block truncatedBlock = arrayBlock.getRegion(startElement, truncatedSlices.length);
    assertBlock(truncatedBlock, truncatedSlices);

    Unnester arrayUnnester = new ArrayUnnester(VARCHAR);
    arrayUnnester.resetInput(truncatedBlock);

    arrayUnnester.startNewOutput(new PageBuilderStatus(), 20);

    // Process all input entries in the truncated block
    for (int i = 0; i < truncatedBlock.getPositionCount(); i++) {
        arrayUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]);
    }

    Block[] output = arrayUnnester.buildOutputBlocksAndFlush();
    assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount());

    Slice[] expectedOutput = computeExpectedUnnestedOutput(truncatedSlices, truncatedUnnestedLengths, 0, truncatedSlices.length);
    assertBlock(output[0], expectedOutput);
}
 
Example #27
Source File: TestLikeFunctions.java    From presto with Apache License 2.0 5 votes vote down vote up
private static Slice offsetHeapSlice(String value)
{
    Slice source = Slices.utf8Slice(value);
    Slice result = Slices.allocate(source.length() + 5);
    result.setBytes(2, source);
    return result.slice(2, source.length());
}
 
Example #28
Source File: BenchmarkRegexpFunctions.java    From presto with Apache License 2.0 5 votes vote down vote up
@Setup
public void setup()
{
    SliceOutput sliceOutput = new DynamicSliceOutput(sourceLength);
    Slice pattern;
    switch (patternString) {
        case ".*x.*":
            pattern = Slices.utf8Slice(".*x.*");
            IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case ".*(x|y).*":
            pattern = Slices.utf8Slice(".*(x|y).*");
            IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "longdotstar":
            pattern = Slices.utf8Slice(".*coolfunctionname.*");
            ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "phone":
            pattern = Slices.utf8Slice("\\d{3}/\\d{3}/\\d{4}");
            // 47: '/', 48-57: '0'-'9'
            ThreadLocalRandom.current().ints(47, 58).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "literal":
            pattern = Slices.utf8Slice("literal");
            // 97-122: 'a'-'z'
            ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        default:
            throw new IllegalArgumentException("pattern: " + patternString + " not supported");
    }

    joniPattern = joniRegexp(pattern);
    re2JPattern = re2JRegexp(pattern);
    source = sliceOutput.slice();
    checkState(source.length() == sourceLength, "source.length=%s, sourceLength=%s", source.length(), sourceLength);
}
 
Example #29
Source File: StringFunctions.java    From presto with Apache License 2.0 5 votes vote down vote up
@ScalarOperator(OperatorType.CAST)
@LiteralParameters("x")
@SqlType(CodePointsType.NAME)
public static int[] castVarcharToCodePoints(@SqlType("varchar(x)") Slice slice)
{
    return castToCodePoints(slice);
}
 
Example #30
Source File: TestCastDependencies.java    From presto with Apache License 2.0 5 votes vote down vote up
@TypeParameter("V")
@SqlType(StandardTypes.VARCHAR)
public static Slice castAnyToVarchar(
        @CastDependency(fromType = "V", toType = StandardTypes.VARCHAR) MethodHandle cast,
        @SqlType("V") long value)
{
    try {
        return (Slice) cast.invokeExact(value);
    }
    catch (Throwable t) {
        throwIfInstanceOf(t, Error.class);
        throwIfInstanceOf(t, PrestoException.class);
        throw new PrestoException(GENERIC_INTERNAL_ERROR, t);
    }
}