io.airlift.slice.Slice Java Examples

@Test
public void test()
{
    Slice[][] expectedValues = new Slice[ARRAY_SIZES.length][];
    for (int i = 0; i < ARRAY_SIZES.length; i++) {
        expectedValues[i] = new Slice[ARRAY_SIZES[i]];
        for (int j = 0; j < ARRAY_SIZES[i]; j++) {
            if (j % 3 != 1) {
                expectedValues[i][j] = Slices.utf8Slice(format("%d.%d", i, j));
            }
        }
    }
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    verifyBlock(blockBuilder, expectedValues);
    verifyBlock(blockBuilder.build(), expectedValues);

    Slice[][] expectedValuesWithNull = alternatingNullValues(expectedValues);
    BlockBuilder blockBuilderWithNull = createBlockBuilderWithValues(expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull, expectedValuesWithNull);
    verifyBlock(blockBuilderWithNull.build(), expectedValuesWithNull);
}
 

public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 

private static DateTimeField getDateField(ISOChronology chronology, Slice unit)
{
    String unitString = unit.toStringUtf8().toLowerCase(ENGLISH);
    switch (unitString) {
        case "day":
            return chronology.dayOfMonth();
        case "week":
            return chronology.weekOfWeekyear();
        case "month":
            return chronology.monthOfYear();
        case "quarter":
            return QUARTER_OF_YEAR.getField(chronology);
        case "year":
            return chronology.year();
    }
    throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "'" + unitString + "' is not a valid DATE field");
}
 

public void testReadUserMetadata()
        throws Exception
{
    try (TempFile tempFile = new TempFile()) {
        Map<String, String> metadata = ImmutableMap.of(
                "a", "ala",
                "b", "ma",
                "c", "kota");
        createFileWithOnlyUserMetadata(tempFile.getFile(), metadata);

        OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE));
        Footer footer = orcReader.getFooter();
        Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii);
        assertEquals(readMetadata, metadata);
    }
}
 

@Override
public final Block decodeColumn(ColumnData columnData)
        throws RcFileCorruptionException
{
    int size = columnData.rowCount();

    Slice slice = columnData.getSlice();
    BlockBuilder builder = type.createBlockBuilder(null, size);
    for (int i = 0; i < size; i++) {
        int length = columnData.getLength(i);
        int offset = columnData.getOffset(i);
        if (!isNullSequence(slice, offset, length)) {
            decodeValueInto(1, builder, slice, offset, length);
        }
        else {
            builder.appendNull();
        }
    }

    return builder.build();
}
 

public Footer(
        long numberOfRows,
        OptionalInt rowsInRowGroup,
        List<StripeInformation> stripes,
        ColumnMetadata<OrcType> types,
        Optional<ColumnMetadata<ColumnStatistics>> fileStats,
        Map<String, Slice> userMetadata)
{
    this.numberOfRows = numberOfRows;
    rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1"));
    this.rowsInRowGroup = rowsInRowGroup;
    this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null"));
    this.types = requireNonNull(types, "types is null");
    this.fileStats = requireNonNull(fileStats, "fileStats is null");
    requireNonNull(userMetadata, "userMetadata is null");
    this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slices::copyOf));
}
 

public DataPageV2(
        int rowCount,
        int nullCount,
        int valueCount,
        Slice repetitionLevels,
        Slice definitionLevels,
        ParquetEncoding dataEncoding,
        Slice slice,
        int uncompressedSize,
        Statistics<?> statistics,
        boolean isCompressed)
{
    super(uncompressedSize, valueCount);
    this.rowCount = rowCount;
    this.nullCount = nullCount;
    this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null");
    this.definitionLevels = requireNonNull(definitionLevels, "definitionLevels slice is null");
    this.dataEncoding = dataEncoding;
    this.slice = requireNonNull(slice, "slice is null");
    this.statistics = statistics;
    this.isCompressed = isCompressed;
}
 

@ScalarFunction
@LiteralParameters({"x", "y"})
@Constraint(variable = "y", expression = "min(2147483647, x + 15)")
// Color formatting uses 15 characters. Note that if the ansiColorEscape function implementation
// changes, this value may be invalidated.
@SqlType("varchar(y)")
public static Slice render(@SqlType("varchar(x)") Slice value, @SqlType(ColorType.NAME) long color)
{
    StringBuilder builder = new StringBuilder(value.length());

    // color
    builder.append(ansiColorEscape(color))
            .append(value.toStringUtf8())
            .append(ANSI_RESET);

    return utf8Slice(builder.toString());
}
 

private Block readLongNotNullBlock()
        throws IOException
{
    verifyNotNull(decimalStream);
    verifyNotNull(scaleStream);

    long[] data = new long[nextBatchSize * 2];
    decimalStream.nextLongDecimal(data, nextBatchSize);

    for (int offset = 0; offset < data.length; offset += 2) {
        long sourceScale = scaleStream.next();
        if (sourceScale != type.getScale()) {
            Slice decimal = Slices.wrappedLongArray(data[offset], data[offset + 1]);
            UnscaledDecimal128Arithmetic.rescale(decimal, (int) (type.getScale() - sourceScale), Slices.wrappedLongArray(data, offset, 2));
        }
    }
    return new Int128ArrayBlock(nextBatchSize, Optional.empty(), data);
}
 

public static void subtract(Slice left, Slice right, Slice result)
{
    if (isNegative(left) != isNegative(right)) {
        // only one is negative
        if (addUnsignedReturnOverflow(left, right, result, isNegative(left)) != 0) {
            throwOverflowException();
        }
    }
    else {
        int compare = compareAbsolute(left, right);
        if (compare > 0) {
            subtractUnsigned(left, right, result, isNegative(left) && isNegative(right));
        }
        else if (compare < 0) {
            subtractUnsigned(right, left, result, !(isNegative(left) && isNegative(right)));
        }
        else {
            setToZero(result);
        }
    }
}
 

public static int patternConstantPrefixBytes(Slice pattern, Optional<Slice> escape)
{
    int escapeChar = getEscapeCharacter(escape)
            .map(c -> (int) c)
            .orElse(-1);

    boolean escaped = false;
    int position = 0;
    while (position < pattern.length()) {
        int currentChar = getCodePointAt(pattern, position);
        if (!escaped && (currentChar == escapeChar)) {
            escaped = true;
        }
        else if (escaped) {
            checkEscape(currentChar == '%' || currentChar == '_' || currentChar == escapeChar);
            escaped = false;
        }
        else if ((currentChar == '%') || (currentChar == '_')) {
            return position;
        }
        position += lengthOfCodePoint(currentChar);
    }
    checkEscape(!escaped);
    return position;
}
 

public static Slice pack(BigInteger unscaledValue, Slice result)
{
    pack(0, 0, false, result);
    byte[] bytes = unscaledValue.abs().toByteArray();

    if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH
            || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) {
        throwOverflowException();
    }

    // convert to little-endian order
    reverse(bytes);

    result.setBytes(0, bytes);
    if (unscaledValue.signum() < 0) {
        setNegative(result, true);
    }

    throwIfOverflows(result);

    return result;
}
 

private LevelReader buildLevelRLEReader(int maxLevel, Slice slice)
{
    if (maxLevel == 0) {
        return new LevelNullReader();
    }
    return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), slice.getInput()));
}
 

public AtopPageSource(Semaphore readerPermits, AtopFactory atopFactory, ConnectorSession session, Slice hostIp, AtopTable table, ZonedDateTime date, List<AtopColumn> columns, List<Type> types)
{
    this.readerPermits = requireNonNull(readerPermits, "readerPermits is null");
    this.atopFactory = requireNonNull(atopFactory, "atopFactory is null");
    this.session = requireNonNull(session, "session is null");
    this.hostIp = requireNonNull(hostIp, "hostIp is null");
    this.table = requireNonNull(table, "table is null");
    this.date = requireNonNull(date, "date is null");
    this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null"));
    this.types = ImmutableList.copyOf(requireNonNull(types, "types is null"));
    checkArgument(columns.size() == types.size(), "columns (%s) does not match types (%s)", columns.size(), types.size());
    pageBuilder = new PageBuilder(types);
}
 

@Test
public void testCopyPositions()
{
    Slice[] expectedValues = alternatingNullValues(createTestValue(17));
    BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues);
    assertBlockFilteredPositions(expectedValues, blockBuilder.build(), () -> blockBuilder.newBlockBuilderLike(null), 0, 2, 4, 6, 7, 9, 10, 16);
}
 

private static void assertDivideAllSigns(Slice dividend, int dividendRescaleFactor, Slice divisor, int divisorRescaleFactor)
{
    assertDivide(dividend, dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(dividend, dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, divisor, divisorRescaleFactor);
    assertDivide(negate(dividend), dividendRescaleFactor, negate(divisor), divisorRescaleFactor);
}
 

@ScalarFunction("match_query")
@Description("es match_query")
@SqlType(StandardTypes.VARCHAR)
@SqlNullable
public static Slice matchQuery(
        @SqlType(StandardTypes.VARCHAR) Slice filter)
{
    if (filter == null) {
        return null;
    }
    String filterStr = filter.toStringUtf8();

    QueryBuilder builder = QueryBuilders.matchQuery(MATCH_COLUMN_SEP, filterStr);
    return Slices.utf8Slice(builder.toString());
}
 

@Override
protected DecimalInputStream createValueStream(Slice slice)
        throws OrcCorruptionException
{
    Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE);
    return new DecimalInputStream(OrcChunkLoader.create(ORC_DATA_SOURCE_ID, slice, orcDecompressor, newSimpleAggregatedMemoryContext()));
}
 

private InputReferenceNode(CallSiteBinder callSiteBinder, Scope scope, Type type, BytecodeExpression block, BytecodeExpression position)
{
    // Generate body based on block and position
    Variable wasNullVariable = scope.getVariable("wasNull");
    Class<?> callType = type.getJavaType();
    if (!callType.isPrimitive() && callType != Slice.class) {
        callType = Object.class;
    }

    IfStatement ifStatement = new IfStatement();
    ifStatement.condition(block.invoke("isNull", boolean.class, position));

    ifStatement.ifTrue()
            .putVariable(wasNullVariable, true)
            .pushJavaDefault(callType);

    String methodName = "get" + Primitives.wrap(callType).getSimpleName();
    BytecodeExpression value = constantType(callSiteBinder, type).invoke(methodName, callType, block, position);
    if (callType != type.getJavaType()) {
        value = value.cast(type.getJavaType());
    }
    ifStatement.ifFalse(value);

    this.body = ifStatement;
    this.block = block;
    this.position = position;
}
 

@Description("Creates a Bing tile from a QuadKey")
@ScalarFunction("bing_tile")
@SqlType(BingTileType.NAME)
public static long toBingTile(@SqlType(StandardTypes.VARCHAR) Slice quadKey)
{
    checkQuadKey(quadKey);
    return BingTile.fromQuadKey(quadKey.toStringUtf8()).encode();
}
 

public Page deserialize(SerializedPage serializedPage)
{
    checkArgument(serializedPage != null, "serializedPage is null");

    Slice slice = serializedPage.getSlice();

    if (serializedPage.isEncrypted()) {
        checkState(spillCipher.isPresent(), "Page is encrypted, but spill cipher is missing");

        byte[] decrypted = new byte[spillCipher.get().decryptedMaxLength(slice.length())];
        int decryptedSize = spillCipher.get().decrypt(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decrypted,
                0);

        slice = Slices.wrappedBuffer(decrypted, 0, decryptedSize);
    }

    if (serializedPage.isCompressed()) {
        checkState(decompressor.isPresent(), "Page is compressed, but decompressor is missing");

        int uncompressedSize = serializedPage.getUncompressedSizeInBytes();
        byte[] decompressed = new byte[uncompressedSize];
        checkState(decompressor.get().decompress(
                slice.byteArray(),
                slice.byteArrayOffset(),
                slice.length(),
                decompressed,
                0,
                uncompressedSize) == uncompressedSize);

        slice = Slices.wrappedBuffer(decompressed);
    }

    return readRawPage(serializedPage.getPositionCount(), slice.getInput(), blockEncodingSerde);
}
 

@ScalarOperator(OperatorType.CAST)
@SqlType("char(y)")
@LiteralParameters({"x", "y"})
public static Slice charToCharCast(@LiteralParameter("x") Long x, @LiteralParameter("y") Long y, @SqlType("char(x)") Slice slice)
{
    if (x > y) {
        return truncateToLength(slice, y.intValue());
    }
    else {
        return slice;
    }
}
 

@LiteralParameters({"p", "s"})
@SqlType("decimal(p, s)")
public static Slice negate(@SqlType("decimal(p, s)") Slice arg)
{
    BigInteger argBigInteger = Decimals.decodeUnscaledValue(arg);
    return encodeUnscaledValue(argBigInteger.negate());
}
 

private <T> Optional<Set<String>> filterString(TupleDomain<T> constraint, T column)
{
    if (constraint.isNone()) {
        return Optional.of(ImmutableSet.of());
    }

    Domain domain = constraint.getDomains().get().get(column);
    if (domain == null) {
        return Optional.empty();
    }

    if (domain.isSingleValue()) {
        return Optional.of(ImmutableSet.of(((Slice) domain.getSingleValue()).toStringUtf8()));
    }
    if (domain.getValues() instanceof EquatableValueSet) {
        Collection<Object> values = ((EquatableValueSet) domain.getValues()).getValues();
        return Optional.of(values.stream()
                .map(Slice.class::cast)
                .map(Slice::toStringUtf8)
                .collect(toImmutableSet()));
    }
    if (domain.getValues() instanceof SortedRangeSet) {
        ImmutableSet.Builder<String> result = ImmutableSet.builder();
        for (Range range : domain.getValues().getRanges().getOrderedRanges()) {
            checkState(!range.isAll()); // Already checked
            if (!range.isSingleValue()) {
                return Optional.empty();
            }

            result.add(((Slice) range.getSingleValue()).toStringUtf8());
        }

        return Optional.of(result.build());
    }
    return Optional.empty();
}
 

@Override
public void decodeValueInto(BlockBuilder builder, Slice slice, int offset, int length)
{
    if (isShortDecimal(type)) {
        type.writeLong(builder, parseLong(slice, offset));
    }
    else {
        type.writeSlice(builder, parseSlice(slice, offset));
    }
}
 

@Test
public void testTrimmedBlocks()
{
    int[] unnestedLengths = {2, 1, 2, 3, 1};

    Slice[][][] elements = column(
            array(toSlices("0.0.0"), toSlices("0.1.0")),
            array(toSlices("1.0.0")),
            array(toSlices("2.0.0"), toSlices("2.1.0")),
            array(toSlices("3.0.0"), toSlices("3.1.0"), toSlices("3.2.0")),
            array(toSlices("4.0.0")));

    Slice[][] slices = getFieldElements(elements, 0);
    Block arrayBlock = createArrayBlock(slices);

    // Remove the first element from the arrayBlock for testing
    int startElement = 1;

    Slice[][] truncatedSlices = Arrays.copyOfRange(slices, startElement, slices.length - startElement + 1);
    int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, slices.length - startElement + 1);
    Block truncatedBlock = arrayBlock.getRegion(startElement, truncatedSlices.length);
    assertBlock(truncatedBlock, truncatedSlices);

    Unnester arrayUnnester = new ArrayUnnester(VARCHAR);
    arrayUnnester.resetInput(truncatedBlock);

    arrayUnnester.startNewOutput(new PageBuilderStatus(), 20);

    // Process all input entries in the truncated block
    for (int i = 0; i < truncatedBlock.getPositionCount(); i++) {
        arrayUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]);
    }

    Block[] output = arrayUnnester.buildOutputBlocksAndFlush();
    assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount());

    Slice[] expectedOutput = computeExpectedUnnestedOutput(truncatedSlices, truncatedUnnestedLengths, 0, truncatedSlices.length);
    assertBlock(output[0], expectedOutput);
}
 

private static Slice offsetHeapSlice(String value)
{
    Slice source = Slices.utf8Slice(value);
    Slice result = Slices.allocate(source.length() + 5);
    result.setBytes(2, source);
    return result.slice(2, source.length());
}
 

@Setup
public void setup()
{
    SliceOutput sliceOutput = new DynamicSliceOutput(sourceLength);
    Slice pattern;
    switch (patternString) {
        case ".*x.*":
            pattern = Slices.utf8Slice(".*x.*");
            IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case ".*(x|y).*":
            pattern = Slices.utf8Slice(".*(x|y).*");
            IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "longdotstar":
            pattern = Slices.utf8Slice(".*coolfunctionname.*");
            ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "phone":
            pattern = Slices.utf8Slice("\\d{3}/\\d{3}/\\d{4}");
            // 47: '/', 48-57: '0'-'9'
            ThreadLocalRandom.current().ints(47, 58).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        case "literal":
            pattern = Slices.utf8Slice("literal");
            // 97-122: 'a'-'z'
            ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte);
            break;
        default:
            throw new IllegalArgumentException("pattern: " + patternString + " not supported");
    }

    joniPattern = joniRegexp(pattern);
    re2JPattern = re2JRegexp(pattern);
    source = sliceOutput.slice();
    checkState(source.length() == sourceLength, "source.length=%s, sourceLength=%s", source.length(), sourceLength);
}
 

@ScalarOperator(OperatorType.CAST)
@LiteralParameters("x")
@SqlType(CodePointsType.NAME)
public static int[] castVarcharToCodePoints(@SqlType("varchar(x)") Slice slice)
{
    return castToCodePoints(slice);
}
 

@TypeParameter("V")
@SqlType(StandardTypes.VARCHAR)
public static Slice castAnyToVarchar(
        @CastDependency(fromType = "V", toType = StandardTypes.VARCHAR) MethodHandle cast,
        @SqlType("V") long value)
{
    try {
        return (Slice) cast.invokeExact(value);
    }
    catch (Throwable t) {
        throwIfInstanceOf(t, Error.class);
        throwIfInstanceOf(t, PrestoException.class);
        throw new PrestoException(GENERIC_INTERNAL_ERROR, t);
    }
}