Java Code Examples for io.airlift.slice.Slice

The following examples show how to use io.airlift.slice.Slice. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: presto   Source File: BlockEncoding.java    License: Apache License 2.0 7 votes vote down vote up
@Override
public final Block decodeColumn(ColumnData columnData)
        throws RcFileCorruptionException
{
    int size = columnData.rowCount();

    Slice slice = columnData.getSlice();
    BlockBuilder builder = type.createBlockBuilder(null, size);
    for (int i = 0; i < size; i++) {
        int length = columnData.getLength(i);
        int offset = columnData.getOffset(i);
        if (!isNullSequence(slice, offset, length)) {
            decodeValueInto(1, builder, slice, offset, length);
        }
        else {
            builder.appendNull();
        }
    }

    return builder.build();
}
 
Example 2
Source Project: presto   Source File: TestColumnarArray.java    License: Apache License 2.0 7 votes vote down vote up
@Test
public void test()
{
    Slice[][] expectedValues = new Slice[ARRAY_SIZES.length][];
    for (int i = 0; i < ARRAY_SIZES.length; i++) {
        expectedValues[i] = new Slice[ARRAY_SIZES[i]];
        for (int j = 0; j < ARRAY_SIZES[i]; j++) {
            if (j % 3 != 1) {
                expectedValues[i][j] = Slices.utf8Slice(format("%d.%d", i, j));
            }
        }
    }
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    verifyBlock(blockBuilder, expectedValues);
    verifyBlock(blockBuilder.build(), expectedValues);

    Slice[][] expectedValuesWithNull = alternatingNullValues(expectedValues);
    BlockBuilder blockBuilderWithNull = createBlockBuilderWithValues(expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull, expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull.build(), expectedValuesWithNull);
}
 
Example 3
Source Project: presto   Source File: DateTimeFunctions.java    License: Apache License 2.0 7 votes vote down vote up
private static DateTimeField getDateField(ISOChronology chronology, Slice unit)
{
    String unitString = unit.toStringUtf8().toLowerCase(ENGLISH);
    switch (unitString) {
        case "day":
            return chronology.dayOfMonth();
        case "week":
            return chronology.weekOfWeekyear();
        case "month":
            return chronology.monthOfYear();
        case "quarter":
            return QUARTER_OF_YEAR.getField(chronology);
        case "year":
            return chronology.year();
    }
    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "'" + unitString + "' is not a valid DATE field");
}
 
Example 4
Source Project: presto   Source File: Chars.java    License: Apache License 2.0 7 votes vote down vote up
public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 
Example 5
public void testReadUserMetadata()
        throws Exception
{
    try (TempFile tempFile = new TempFile()) {
        Map<String, String> metadata = ImmutableMap.of(
                "a", "ala",
                "b", "ma",
                "c", "kota");
        createFileWithOnlyUserMetadata(tempFile.getFile(), metadata);

        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        Footer footer = orcReader.getFooter();
        Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii);
        assertEquals(readMetadata, metadata);
    }
}
 
Example 6
Source Project: presto   Source File: DataPageV2.java    License: Apache License 2.0 6 votes vote down vote up
public DataPageV2(
        int rowCount,
        int nullCount,
        int valueCount,
        Slice repetitionLevels,
        Slice definitionLevels,
        ParquetEncoding dataEncoding,
        Slice slice,
        int uncompressedSize,
        Statistics<?> statistics,
        boolean isCompressed)
{
    super(uncompressedSize, valueCount);
    this.rowCount = rowCount;
    this.nullCount = nullCount;
    this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
    this.definitionLevels = requireNonNull(definitionLevels, "definitionLevels slice is null");
    this.dataEncoding = dataEncoding;
    this.slice = requireNonNull(slice, "slice is null");
    this.statistics = statistics;
    this.isCompressed = isCompressed;
}
 
Example 7
Source Project: presto   Source File: ColorFunctions.java    License: Apache License 2.0 6 votes vote down vote up
@ScalarFunction
@LiteralParameters({"x", "y"})
@Constraint(variable = "y", expression = "min(2147483647, x + 15)")
// Color formatting uses 15 characters. Note that if the ansiColorEscape function implementation
// changes, this value may be invalidated.
@SqlType("varchar(y)")
public static Slice render(@SqlType("varchar(x)") Slice value, @SqlType(ColorType.NAME) long color)
{
    StringBuilder builder = new StringBuilder(value.length());

    // color
    builder.append(ansiColorEscape(color))
            .append(value.toStringUtf8())
            .append(ANSI_RESET);

    return utf8Slice(builder.toString());
}
 
Example 8
Source Project: presto   Source File: LikeFunctions.java    License: Apache License 2.0 6 votes vote down vote up
public static int patternConstantPrefixBytes(Slice pattern, Optional<Slice> escape)
{
    int escapeChar = getEscapeCharacter(escape)
            .map(c -> (int) c)
            .orElse(-1);

    boolean escaped = false;
    int position = 0;
    while (position < pattern.length()) {
        int currentChar = getCodePointAt(pattern, position);
        if (!escaped && (currentChar == escapeChar)) {
            escaped = true;
        }
        else if (escaped) {
            checkEscape(currentChar == '%' || currentChar == '_' || currentChar == escapeChar);
            escaped = false;
        }
        else if ((currentChar == '%') || (currentChar == '_')) {
            return position;
        }
        position += lengthOfCodePoint(currentChar);
    }
    checkEscape(!escaped);
    return position;
}
 
Example 9
Source Project: presto   Source File: Footer.java    License: Apache License 2.0 6 votes vote down vote up
public Footer(
        long numberOfRows,
        OptionalInt rowsInRowGroup,
        List<StripeInformation> stripes,
        ColumnMetadata<OrcType> types,
        Optional<ColumnMetadata<ColumnStatistics>> fileStats,
        Map<String, Slice> userMetadata)
{
    this.numberOfRows = numberOfRows;
    rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1"));
    this.rowsInRowGroup = rowsInRowGroup;
    this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null"));
    this.types = requireNonNull(types, "types is null");
    this.fileStats = requireNonNull(fileStats, "fileStats is null");
    requireNonNull(userMetadata, "userMetadata is null");
    this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slices::copyOf));
}
 
Example 10
Source Project: presto   Source File: DecimalColumnReader.java    License: Apache License 2.0 6 votes vote down vote up
private Block readLongNotNullBlock()
        throws IOException
{
    verifyNotNull(decimalStream);
    verifyNotNull(scaleStream);

    long[] data = new long[nextBatchSize * 2];
    decimalStream.nextLongDecimal(data, nextBatchSize);

    for (int offset = 0; offset < data.length; offset += 2) {
        long sourceScale = scaleStream.next();
        if (sourceScale != type.getScale()) {
            Slice decimal = Slices.wrappedLongArray(data[offset], data[offset + 1]);
            UnscaledDecimal128Arithmetic.rescale(decimal, (int) (type.getScale() - sourceScale), Slices.wrappedLongArray(data, offset, 2));
        }
    }
    return new Int128ArrayBlock(nextBatchSize, Optional.empty(), data);
}
 
Example 11
Source Project: presto   Source File: UnscaledDecimal128Arithmetic.java    License: Apache License 2.0 6 votes vote down vote up
public static Slice pack(BigInteger unscaledValue, Slice result)
{
    pack(0, 0, false, result);
    byte[] bytes = unscaledValue.abs().toByteArray();

    if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH
            || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) {
        throwOverflowException();
    }

    // convert to little-endian order
    reverse(bytes);

    result.setBytes(0, bytes);
    if (unscaledValue.signum() < 0) {
        setNegative(result, true);
    }

    throwIfOverflows(result);

    return result;
}
 
Example 12
Source Project: presto   Source File: UnscaledDecimal128Arithmetic.java    License: Apache License 2.0 6 votes vote down vote up
public static void subtract(Slice left, Slice right, Slice result)
{
    if (isNegative(left) != isNegative(right)) {
        // only one is negative
        if (addUnsignedReturnOverflow(left, right, result, isNegative(left)) != 0) {
            throwOverflowException();
        }
    }
    else {
        int compare = compareAbsolute(left, right);
        if (compare > 0) {
            subtractUnsigned(left, right, result, isNegative(left) && isNegative(right));
        }
        else if (compare < 0) {
            subtractUnsigned(right, left, result, !(isNegative(left) && isNegative(right)));
        }
        else {
            setToZero(result);
        }
    }
}
 
Example 13
Source Project: presto   Source File: DateTimeFunctions.java    License: Apache License 2.0 5 votes vote down vote up
@Description("Difference of the given dates in the given unit")
@ScalarFunction("date_diff")
@LiteralParameters("x")
@SqlType(StandardTypes.BIGINT)
public static long diffDate(ConnectorSession session, @SqlType("varchar(x)") Slice unit, @SqlType(StandardTypes.DATE) long date1, @SqlType(StandardTypes.DATE) long date2)
{
    return getDateField(UTC_CHRONOLOGY, unit).getDifferenceAsLong(DAYS.toMillis(date2), DAYS.toMillis(date1));
}
 
Example 14
Source Project: presto   Source File: PreparedStatementBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private static Object getBindValue(int columnIndex, Set<Integer> uuidColumnIndexes, Object value)
{
    if (uuidColumnIndexes.contains(columnIndex)) {
        return uuidToBytes(fromString(((Slice) value).toStringUtf8()));
    }
    return value;
}
 
Example 15
Source Project: presto   Source File: FilterUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static <T> Optional<String> tryGetSingleVarcharValue(TupleDomain<T> constraint, T index)
{
    if (constraint.isNone()) {
        return Optional.empty();
    }

    Domain domain = constraint.getDomains().get().get(index);
    if ((domain == null) || !domain.isSingleValue()) {
        return Optional.empty();
    }

    Object value = domain.getSingleValue();
    return Optional.of(((Slice) value).toStringUtf8());
}
 
Example 16
@InputFunction
public static void input(SpatialPartitioningState state, @SqlType(GEOMETRY_TYPE_NAME) Slice slice, @SqlType(INTEGER) long partitionCount)
{
    Envelope envelope = deserializeEnvelope(slice);
    if (envelope.isEmpty()) {
        return;
    }

    Rectangle extent = new Rectangle(envelope.getXMin(), envelope.getYMin(), envelope.getXMax(), envelope.getYMax());

    if (state.getCount() == 0) {
        state.setPartitionCount(toIntExact(partitionCount));
        state.setExtent(extent);
        state.setSamples(new ArrayList<>());
    }
    else {
        state.setExtent(state.getExtent().merge(extent));
    }

    // use reservoir sampling
    List<Rectangle> samples = state.getSamples();
    if (samples.size() <= MAX_SAMPLE_COUNT) {
        samples.add(extent);
    }
    else {
        long sampleIndex = ThreadLocalRandom.current().nextLong(state.getCount());
        if (sampleIndex < MAX_SAMPLE_COUNT) {
            samples.set(toIntExact(sampleIndex), extent);
        }
    }

    state.setCount(state.getCount() + 1);
}
 
Example 17
private static int[] castToCodePoints(Slice slice) throws HiveException {
    int[] codePoints = new int[safeCountCodePoints(slice)];
    int position = 0;
    for (int index = 0; index < codePoints.length; index++) {
        codePoints[index] = getCodePointAt(slice, position);
        position += lengthOfCodePoint(slice, position);
    }
    return codePoints;
}
 
Example 18
Source Project: presto   Source File: BingTileFunctions.java    License: Apache License 2.0 5 votes vote down vote up
@Description("Creates a Bing tile from a QuadKey")
@ScalarFunction("bing_tile")
@SqlType(BingTileType.NAME)
public static long toBingTile(@SqlType(StandardTypes.VARCHAR) Slice quadKey)
{
    checkQuadKey(quadKey);
    return BingTile.fromQuadKey(quadKey.toStringUtf8()).encode();
}
 
Example 19
Source Project: presto   Source File: VarbinaryFunctions.java    License: Apache License 2.0 5 votes vote down vote up
@Description("Compute md5 hash")
@ScalarFunction
@SqlType(StandardTypes.VARBINARY)
public static Slice md5(@SqlType(StandardTypes.VARBINARY) Slice slice)
{
    return computeHash(Hashing.md5(), slice);
}
 
Example 20
Source Project: presto   Source File: TeradataDateFunctions.java    License: Apache License 2.0 5 votes vote down vote up
@Description("Converts a string to a TIMESTAMP data type")
@ScalarFunction("to_timestamp")
@SqlType("timestamp(3)")
public static long toTimestamp(
        ConnectorSession session,
        @SqlType(StandardTypes.VARCHAR) Slice dateTime,
        @SqlType(StandardTypes.VARCHAR) Slice formatString)
{
    return parseMillis(session, dateTime, formatString);
}
 
Example 21
Source Project: presto   Source File: StringFunctions.java    License: Apache License 2.0 5 votes vote down vote up
@Description("Suffix starting at given index")
@ScalarFunction(value = "substring", alias = "substr")
@LiteralParameters("x")
@SqlType("varchar(x)")
public static Slice charSubstring(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice utf8, @SqlType(StandardTypes.BIGINT) long start)
{
    return substring(padSpaces(utf8, x.intValue()), start);
}
 
Example 22
Source Project: presto-kudu   Source File: KuduRecordCursor.java    License: Apache License 2.0 5 votes vote down vote up
private int getRowLength() {
    org.apache.kudu.util.Slice rawData = getCurrentRowRawData();
    if (rawData != null) {
        return rawData.length();
    } else {
        return columnTypes.size();
    }
}
 
Example 23
Source Project: presto   Source File: TestChars.java    License: Apache License 2.0 5 votes vote down vote up
private static void assertByteCountWithoutTrailingSpace(byte[] actual, int offset, int length, byte[] expected)
{
    Slice slice = wrappedBuffer(actual);
    int trimmedLength = byteCountWithoutTrailingSpace(slice, offset, length);
    byte[] bytes = slice.getBytes(offset, trimmedLength);
    assertEquals(bytes, expected);
}
 
Example 24
Source Project: presto   Source File: BinaryEncoding.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encodeValueInto(int depth, Block block, int position, SliceOutput output)
{
    Slice slice = type.getSlice(block, position);
    byte[] data = slice.getBytes();
    slice = Slices.wrappedBuffer(base64Encoder.encode(data));
    output.writeBytes(slice);
}
 
Example 25
Source Project: presto   Source File: InputReferenceCompiler.java    License: Apache License 2.0 5 votes vote down vote up
private InputReferenceNode(CallSiteBinder callSiteBinder, Scope scope, Type type, BytecodeExpression block, BytecodeExpression position)
{
    // Generate body based on block and position
    Variable wasNullVariable = scope.getVariable("wasNull");
    Class<?> callType = type.getJavaType();
    if (!callType.isPrimitive() && callType != Slice.class) {
        callType = Object.class;
    }

    IfStatement ifStatement = new IfStatement();
    ifStatement.condition(block.invoke("isNull", boolean.class, position));

    ifStatement.ifTrue()
            .putVariable(wasNullVariable, true)
            .pushJavaDefault(callType);

    String methodName = "get" + Primitives.wrap(callType).getSimpleName();
    BytecodeExpression value = constantType(callSiteBinder, type).invoke(methodName, callType, block, position);
    if (callType != type.getJavaType()) {
        value = value.cast(type.getJavaType());
    }
    ifStatement.ifFalse(value);

    this.body = ifStatement;
    this.block = block;
    this.position = position;
}
 
Example 26
Source Project: presto   Source File: PrimitiveColumnReader.java    License: Apache License 2.0 5 votes vote down vote up
private LevelReader buildLevelRLEReader(int maxLevel, Slice slice)
{
    if (maxLevel == 0) {
        return new LevelNullReader();
    }
    return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), slice.getInput()));
}
 
Example 27
Source Project: presto   Source File: TestUnscaledDecimal128Arithmetic.java    License: Apache License 2.0 5 votes vote down vote up
private static void assertDivideAllSigns(Slice dividend, int dividendRescaleFactor, Slice divisor, int divisorRescaleFactor)
{
    assertDivide(dividend, dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(dividend, dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
}
 
Example 28
Source Project: presto   Source File: TestInt128ArrayBlock.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testCopyPositions()
{
    Slice[] expectedValues = alternatingNullValues(createTestValue(17));
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    assertBlockFilteredPositions(expectedValues, blockBuilder.build(), () -> blockBuilder.newBlockBuilderLike(null), 0, 2, 4, 6, 7, 9, 10, 16);
}
 
Example 29
Source Project: presto   Source File: AtopPageSource.java    License: Apache License 2.0 5 votes vote down vote up
public AtopPageSource(Semaphore readerPermits, AtopFactory atopFactory, ConnectorSession session, Slice hostIp, AtopTable table, ZonedDateTime date, List<AtopColumn> columns, List<Type> types)
{
    this.readerPermits = requireNonNull(readerPermits, "readerPermits is null");
    this.atopFactory = requireNonNull(atopFactory, "atopFactory is null");
    this.session = requireNonNull(session, "session is null");
    this.hostIp = requireNonNull(hostIp, "hostIp is null");
    this.table = requireNonNull(table, "table is null");
    this.date = requireNonNull(date, "date is null");
    this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
    this.types = ImmutableList.copyOf(requireNonNull(types, "types is null"));
    checkArgument(columns.size() == types.size(), "columns (%s) does not match types (%s)", columns.size(), types.size());
    pageBuilder = new PageBuilder(types);
}
 
Example 30
Source Project: presto   Source File: PagesSerde.java    License: Apache License 2.0 5 votes vote down vote up
public Page deserialize(SerializedPage serializedPage)
{
    checkArgument(serializedPage != null, "serializedPage is null");

    Slice slice = serializedPage.getSlice();

    if (serializedPage.isEncrypted()) {
        checkState(spillCipher.isPresent(), "Page is encrypted, but spill cipher is missing");

        byte[] decrypted = new byte[spillCipher.get().decryptedMaxLength(slice.length())];
        int decryptedSize = spillCipher.get().decrypt(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decrypted,
                0);

        slice = Slices.wrappedBuffer(decrypted, 0, decryptedSize);
    }

    if (serializedPage.isCompressed()) {
        checkState(decompressor.isPresent(), "Page is compressed, but decompressor is missing");

        int uncompressedSize = serializedPage.getUncompressedSizeInBytes();
        byte[] decompressed = new byte[uncompressedSize];
        checkState(decompressor.get().decompress(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decompressed,
                0,
                uncompressedSize) == uncompressedSize);

        slice = Slices.wrappedBuffer(decompressed);
    }

    return readRawPage(serializedPage.getPositionCount(), slice.getInput(), blockEncodingSerde);
}