io.airlift.slice.Slice Java Examples
The following examples show how to use
io.airlift.slice.Slice.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestColumnarArray.java From presto with Apache License 2.0 | 7 votes |
@Test public void test() { Slice[][] expectedValues = new Slice[ARRAY_SIZES.length][]; for (int i = 0; i < ARRAY_SIZES.length; i++) { expectedValues[i] = new Slice[ARRAY_SIZES[i]]; for (int j = 0; j < ARRAY_SIZES[i]; j++) { if (j % 3 != 1) { expectedValues[i][j] = Slices.utf8Slice(format("%d.%d", i, j)); } } } BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues); verifyBlock(blockBuilder, expectedValues); verifyBlock(blockBuilder.build(), expectedValues); Slice[][] expectedValuesWithNull = alternatingNullValues(expectedValues); BlockBuilder blockBuilderWithNull = createBlockBuilderWithValues(expectedValuesWithNull); verifyBlock(blockBuilderWithNull, expectedValuesWithNull); verifyBlock(blockBuilderWithNull.build(), expectedValuesWithNull); }
Example #2
Source File: Chars.java From presto with Apache License 2.0 | 7 votes |
public static Slice padSpaces(Slice slice, int length) { int textLength = countCodePoints(slice); if (textLength > length) { throw new IllegalArgumentException("pad length is smaller than slice length"); } if (textLength == length) { return slice; } int bufferSize = slice.length() + length - textLength; Slice buffer = Slices.allocate(bufferSize); buffer.setBytes(0, slice); for (int i = slice.length(); i < bufferSize; ++i) { buffer.setByte(i, ' '); } return buffer; }
Example #3
Source File: DateTimeFunctions.java From presto with Apache License 2.0 | 7 votes |
private static DateTimeField getDateField(ISOChronology chronology, Slice unit) { String unitString = unit.toStringUtf8().toLowerCase(ENGLISH); switch (unitString) { case "day": return chronology.dayOfMonth(); case "week": return chronology.weekOfWeekyear(); case "month": return chronology.monthOfYear(); case "quarter": return QUARTER_OF_YEAR.getField(chronology); case "year": return chronology.year(); } throw new PrestoException(INVALID_FUNCTION_ARGUMENT, "'" + unitString + "' is not a valid DATE field"); }
Example #4
Source File: TestOrcReaderPositions.java From spliceengine with GNU Affero General Public License v3.0 | 7 votes |
public void testReadUserMetadata() throws Exception { try (TempFile tempFile = new TempFile()) { Map<String, String> metadata = ImmutableMap.of( "a", "ala", "b", "ma", "c", "kota"); createFileWithOnlyUserMetadata(tempFile.getFile(), metadata); OrcDataSource orcDataSource = new FileOrcDataSource(tempFile.getFile(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE)); OrcReader orcReader = new OrcReader(orcDataSource, new OrcMetadataReader(), new DataSize(1, DataSize.Unit.MEGABYTE), new DataSize(1, DataSize.Unit.MEGABYTE)); Footer footer = orcReader.getFooter(); Map<String, String> readMetadata = Maps.transformValues(footer.getUserMetadata(), Slice::toStringAscii); assertEquals(readMetadata, metadata); } }
Example #5
Source File: BlockEncoding.java From presto with Apache License 2.0 | 7 votes |
@Override public final Block decodeColumn(ColumnData columnData) throws RcFileCorruptionException { int size = columnData.rowCount(); Slice slice = columnData.getSlice(); BlockBuilder builder = type.createBlockBuilder(null, size); for (int i = 0; i < size; i++) { int length = columnData.getLength(i); int offset = columnData.getOffset(i); if (!isNullSequence(slice, offset, length)) { decodeValueInto(1, builder, slice, offset, length); } else { builder.appendNull(); } } return builder.build(); }
Example #6
Source File: Footer.java From presto with Apache License 2.0 | 6 votes |
public Footer( long numberOfRows, OptionalInt rowsInRowGroup, List<StripeInformation> stripes, ColumnMetadata<OrcType> types, Optional<ColumnMetadata<ColumnStatistics>> fileStats, Map<String, Slice> userMetadata) { this.numberOfRows = numberOfRows; rowsInRowGroup.ifPresent(value -> checkArgument(value > 0, "rowsInRowGroup must be at least 1")); this.rowsInRowGroup = rowsInRowGroup; this.stripes = ImmutableList.copyOf(requireNonNull(stripes, "stripes is null")); this.types = requireNonNull(types, "types is null"); this.fileStats = requireNonNull(fileStats, "fileStats is null"); requireNonNull(userMetadata, "userMetadata is null"); this.userMetadata = ImmutableMap.copyOf(transformValues(userMetadata, Slices::copyOf)); }
Example #7
Source File: DataPageV2.java From presto with Apache License 2.0 | 6 votes |
public DataPageV2( int rowCount, int nullCount, int valueCount, Slice repetitionLevels, Slice definitionLevels, ParquetEncoding dataEncoding, Slice slice, int uncompressedSize, Statistics<?> statistics, boolean isCompressed) { super(uncompressedSize, valueCount); this.rowCount = rowCount; this.nullCount = nullCount; this.repetitionLevels = requireNonNull(repetitionLevels, "repetitionLevels slice is null"); this.definitionLevels = requireNonNull(definitionLevels, "definitionLevels slice is null"); this.dataEncoding = dataEncoding; this.slice = requireNonNull(slice, "slice is null"); this.statistics = statistics; this.isCompressed = isCompressed; }
Example #8
Source File: ColorFunctions.java From presto with Apache License 2.0 | 6 votes |
@ScalarFunction @LiteralParameters({"x", "y"}) @Constraint(variable = "y", expression = "min(2147483647, x + 15)") // Color formatting uses 15 characters. Note that if the ansiColorEscape function implementation // changes, this value may be invalidated. @SqlType("varchar(y)") public static Slice render(@SqlType("varchar(x)") Slice value, @SqlType(ColorType.NAME) long color) { StringBuilder builder = new StringBuilder(value.length()); // color builder.append(ansiColorEscape(color)) .append(value.toStringUtf8()) .append(ANSI_RESET); return utf8Slice(builder.toString()); }
Example #9
Source File: DecimalColumnReader.java From presto with Apache License 2.0 | 6 votes |
private Block readLongNotNullBlock() throws IOException { verifyNotNull(decimalStream); verifyNotNull(scaleStream); long[] data = new long[nextBatchSize * 2]; decimalStream.nextLongDecimal(data, nextBatchSize); for (int offset = 0; offset < data.length; offset += 2) { long sourceScale = scaleStream.next(); if (sourceScale != type.getScale()) { Slice decimal = Slices.wrappedLongArray(data[offset], data[offset + 1]); UnscaledDecimal128Arithmetic.rescale(decimal, (int) (type.getScale() - sourceScale), Slices.wrappedLongArray(data, offset, 2)); } } return new Int128ArrayBlock(nextBatchSize, Optional.empty(), data); }
Example #10
Source File: UnscaledDecimal128Arithmetic.java From presto with Apache License 2.0 | 6 votes |
public static void subtract(Slice left, Slice right, Slice result) { if (isNegative(left) != isNegative(right)) { // only one is negative if (addUnsignedReturnOverflow(left, right, result, isNegative(left)) != 0) { throwOverflowException(); } } else { int compare = compareAbsolute(left, right); if (compare > 0) { subtractUnsigned(left, right, result, isNegative(left) && isNegative(right)); } else if (compare < 0) { subtractUnsigned(right, left, result, !(isNegative(left) && isNegative(right))); } else { setToZero(result); } } }
Example #11
Source File: LikeFunctions.java From presto with Apache License 2.0 | 6 votes |
public static int patternConstantPrefixBytes(Slice pattern, Optional<Slice> escape) { int escapeChar = getEscapeCharacter(escape) .map(c -> (int) c) .orElse(-1); boolean escaped = false; int position = 0; while (position < pattern.length()) { int currentChar = getCodePointAt(pattern, position); if (!escaped && (currentChar == escapeChar)) { escaped = true; } else if (escaped) { checkEscape(currentChar == '%' || currentChar == '_' || currentChar == escapeChar); escaped = false; } else if ((currentChar == '%') || (currentChar == '_')) { return position; } position += lengthOfCodePoint(currentChar); } checkEscape(!escaped); return position; }
Example #12
Source File: UnscaledDecimal128Arithmetic.java From presto with Apache License 2.0 | 6 votes |
public static Slice pack(BigInteger unscaledValue, Slice result) { pack(0, 0, false, result); byte[] bytes = unscaledValue.abs().toByteArray(); if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) { throwOverflowException(); } // convert to little-endian order reverse(bytes); result.setBytes(0, bytes); if (unscaledValue.signum() < 0) { setNegative(result, true); } throwIfOverflows(result); return result; }
Example #13
Source File: PrimitiveColumnReader.java From presto with Apache License 2.0 | 5 votes |
private LevelReader buildLevelRLEReader(int maxLevel, Slice slice) { if (maxLevel == 0) { return new LevelNullReader(); } return new LevelRLEReader(new RunLengthBitPackingHybridDecoder(BytesUtils.getWidthFromMaxInt(maxLevel), slice.getInput())); }
Example #14
Source File: AtopPageSource.java From presto with Apache License 2.0 | 5 votes |
public AtopPageSource(Semaphore readerPermits, AtopFactory atopFactory, ConnectorSession session, Slice hostIp, AtopTable table, ZonedDateTime date, List<AtopColumn> columns, List<Type> types) { this.readerPermits = requireNonNull(readerPermits, "readerPermits is null"); this.atopFactory = requireNonNull(atopFactory, "atopFactory is null"); this.session = requireNonNull(session, "session is null"); this.hostIp = requireNonNull(hostIp, "hostIp is null"); this.table = requireNonNull(table, "table is null"); this.date = requireNonNull(date, "date is null"); this.columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); this.types = ImmutableList.copyOf(requireNonNull(types, "types is null")); checkArgument(columns.size() == types.size(), "columns (%s) does not match types (%s)", columns.size(), types.size()); pageBuilder = new PageBuilder(types); }
Example #15
Source File: TestInt128ArrayBlock.java From presto with Apache License 2.0 | 5 votes |
@Test public void testCopyPositions() { Slice[] expectedValues = alternatingNullValues(createTestValue(17)); BlockBuilder blockBuilder = createBlockBuilderWithValues(expectedValues); assertBlockFilteredPositions(expectedValues, blockBuilder.build(), () -> blockBuilder.newBlockBuilderLike(null), 0, 2, 4, 6, 7, 9, 10, 16); }
Example #16
Source File: TestUnscaledDecimal128Arithmetic.java From presto with Apache License 2.0 | 5 votes |
private static void assertDivideAllSigns(Slice dividend, int dividendRescaleFactor, Slice divisor, int divisorRescaleFactor) { assertDivide(dividend, dividendRescaleFactor, divisor, divisorRescaleFactor); assertDivide(dividend, dividendRescaleFactor, negate(divisor), divisorRescaleFactor); assertDivide(negate(dividend), dividendRescaleFactor, divisor, divisorRescaleFactor); assertDivide(negate(dividend), dividendRescaleFactor, negate(divisor), divisorRescaleFactor); }
Example #17
Source File: MatchQueryFunction.java From presto-connectors with Apache License 2.0 | 5 votes |
@ScalarFunction("match_query") @Description("es match_query") @SqlType(StandardTypes.VARCHAR) @SqlNullable public static Slice matchQuery( @SqlType(StandardTypes.VARCHAR) Slice filter) { if (filter == null) { return null; } String filterStr = filter.toStringUtf8(); QueryBuilder builder = QueryBuilders.matchQuery(MATCH_COLUMN_SEP, filterStr); return Slices.utf8Slice(builder.toString()); }
Example #18
Source File: TestLongDecimalStream.java From presto with Apache License 2.0 | 5 votes |
@Override protected DecimalInputStream createValueStream(Slice slice) throws OrcCorruptionException { Optional<OrcDecompressor> orcDecompressor = createOrcDecompressor(ORC_DATA_SOURCE_ID, SNAPPY, COMPRESSION_BLOCK_SIZE); return new DecimalInputStream(OrcChunkLoader.create(ORC_DATA_SOURCE_ID, slice, orcDecompressor, newSimpleAggregatedMemoryContext())); }
Example #19
Source File: InputReferenceCompiler.java From presto with Apache License 2.0 | 5 votes |
private InputReferenceNode(CallSiteBinder callSiteBinder, Scope scope, Type type, BytecodeExpression block, BytecodeExpression position) { // Generate body based on block and position Variable wasNullVariable = scope.getVariable("wasNull"); Class<?> callType = type.getJavaType(); if (!callType.isPrimitive() && callType != Slice.class) { callType = Object.class; } IfStatement ifStatement = new IfStatement(); ifStatement.condition(block.invoke("isNull", boolean.class, position)); ifStatement.ifTrue() .putVariable(wasNullVariable, true) .pushJavaDefault(callType); String methodName = "get" + Primitives.wrap(callType).getSimpleName(); BytecodeExpression value = constantType(callSiteBinder, type).invoke(methodName, callType, block, position); if (callType != type.getJavaType()) { value = value.cast(type.getJavaType()); } ifStatement.ifFalse(value); this.body = ifStatement; this.block = block; this.position = position; }
Example #20
Source File: BingTileFunctions.java From presto with Apache License 2.0 | 5 votes |
@Description("Creates a Bing tile from a QuadKey") @ScalarFunction("bing_tile") @SqlType(BingTileType.NAME) public static long toBingTile(@SqlType(StandardTypes.VARCHAR) Slice quadKey) { checkQuadKey(quadKey); return BingTile.fromQuadKey(quadKey.toStringUtf8()).encode(); }
Example #21
Source File: PagesSerde.java From presto with Apache License 2.0 | 5 votes |
public Page deserialize(SerializedPage serializedPage) { checkArgument(serializedPage != null, "serializedPage is null"); Slice slice = serializedPage.getSlice(); if (serializedPage.isEncrypted()) { checkState(spillCipher.isPresent(), "Page is encrypted, but spill cipher is missing"); byte[] decrypted = new byte[spillCipher.get().decryptedMaxLength(slice.length())]; int decryptedSize = spillCipher.get().decrypt( slice.byteArray(), slice.byteArrayOffset(), slice.length(), decrypted, 0); slice = Slices.wrappedBuffer(decrypted, 0, decryptedSize); } if (serializedPage.isCompressed()) { checkState(decompressor.isPresent(), "Page is compressed, but decompressor is missing"); int uncompressedSize = serializedPage.getUncompressedSizeInBytes(); byte[] decompressed = new byte[uncompressedSize]; checkState(decompressor.get().decompress( slice.byteArray(), slice.byteArrayOffset(), slice.length(), decompressed, 0, uncompressedSize) == uncompressedSize); slice = Slices.wrappedBuffer(decompressed); } return readRawPage(serializedPage.getPositionCount(), slice.getInput(), blockEncodingSerde); }
Example #22
Source File: CharacterStringCasts.java From presto with Apache License 2.0 | 5 votes |
@ScalarOperator(OperatorType.CAST) @SqlType("char(y)") @LiteralParameters({"x", "y"}) public static Slice charToCharCast(@LiteralParameter("x") Long x, @LiteralParameter("y") Long y, @SqlType("char(x)") Slice slice) { if (x > y) { return truncateToLength(slice, y.intValue()); } else { return slice; } }
Example #23
Source File: DecimalOperators.java From presto with Apache License 2.0 | 5 votes |
@LiteralParameters({"p", "s"}) @SqlType("decimal(p, s)") public static Slice negate(@SqlType("decimal(p, s)") Slice arg) { BigInteger argBigInteger = Decimals.decodeUnscaledValue(arg); return encodeUnscaledValue(argBigInteger.negate()); }
Example #24
Source File: InformationSchemaMetadata.java From presto with Apache License 2.0 | 5 votes |
private <T> Optional<Set<String>> filterString(TupleDomain<T> constraint, T column) { if (constraint.isNone()) { return Optional.of(ImmutableSet.of()); } Domain domain = constraint.getDomains().get().get(column); if (domain == null) { return Optional.empty(); } if (domain.isSingleValue()) { return Optional.of(ImmutableSet.of(((Slice) domain.getSingleValue()).toStringUtf8())); } if (domain.getValues() instanceof EquatableValueSet) { Collection<Object> values = ((EquatableValueSet) domain.getValues()).getValues(); return Optional.of(values.stream() .map(Slice.class::cast) .map(Slice::toStringUtf8) .collect(toImmutableSet())); } if (domain.getValues() instanceof SortedRangeSet) { ImmutableSet.Builder<String> result = ImmutableSet.builder(); for (Range range : domain.getValues().getRanges().getOrderedRanges()) { checkState(!range.isAll()); // Already checked if (!range.isSingleValue()) { return Optional.empty(); } result.add(((Slice) range.getSingleValue()).toStringUtf8()); } return Optional.of(result.build()); } return Optional.empty(); }
Example #25
Source File: DecimalEncoding.java From presto with Apache License 2.0 | 5 votes |
@Override public void decodeValueInto(BlockBuilder builder, Slice slice, int offset, int length) { if (isShortDecimal(type)) { type.writeLong(builder, parseLong(slice, offset)); } else { type.writeSlice(builder, parseSlice(slice, offset)); } }
Example #26
Source File: TestArrayUnnester.java From presto with Apache License 2.0 | 5 votes |
@Test public void testTrimmedBlocks() { int[] unnestedLengths = {2, 1, 2, 3, 1}; Slice[][][] elements = column( array(toSlices("0.0.0"), toSlices("0.1.0")), array(toSlices("1.0.0")), array(toSlices("2.0.0"), toSlices("2.1.0")), array(toSlices("3.0.0"), toSlices("3.1.0"), toSlices("3.2.0")), array(toSlices("4.0.0"))); Slice[][] slices = getFieldElements(elements, 0); Block arrayBlock = createArrayBlock(slices); // Remove the first element from the arrayBlock for testing int startElement = 1; Slice[][] truncatedSlices = Arrays.copyOfRange(slices, startElement, slices.length - startElement + 1); int[] truncatedUnnestedLengths = Arrays.copyOfRange(unnestedLengths, startElement, slices.length - startElement + 1); Block truncatedBlock = arrayBlock.getRegion(startElement, truncatedSlices.length); assertBlock(truncatedBlock, truncatedSlices); Unnester arrayUnnester = new ArrayUnnester(VARCHAR); arrayUnnester.resetInput(truncatedBlock); arrayUnnester.startNewOutput(new PageBuilderStatus(), 20); // Process all input entries in the truncated block for (int i = 0; i < truncatedBlock.getPositionCount(); i++) { arrayUnnester.processCurrentAndAdvance(truncatedUnnestedLengths[i]); } Block[] output = arrayUnnester.buildOutputBlocksAndFlush(); assertEquals(Arrays.asList(truncatedSlices).stream().mapToInt(slice -> slice.length).sum(), output[0].getPositionCount()); Slice[] expectedOutput = computeExpectedUnnestedOutput(truncatedSlices, truncatedUnnestedLengths, 0, truncatedSlices.length); assertBlock(output[0], expectedOutput); }
Example #27
Source File: TestLikeFunctions.java From presto with Apache License 2.0 | 5 votes |
private static Slice offsetHeapSlice(String value) { Slice source = Slices.utf8Slice(value); Slice result = Slices.allocate(source.length() + 5); result.setBytes(2, source); return result.slice(2, source.length()); }
Example #28
Source File: BenchmarkRegexpFunctions.java From presto with Apache License 2.0 | 5 votes |
@Setup public void setup() { SliceOutput sliceOutput = new DynamicSliceOutput(sourceLength); Slice pattern; switch (patternString) { case ".*x.*": pattern = Slices.utf8Slice(".*x.*"); IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte); break; case ".*(x|y).*": pattern = Slices.utf8Slice(".*(x|y).*"); IntStream.generate(() -> 97).limit(sourceLength).forEach(sliceOutput::appendByte); break; case "longdotstar": pattern = Slices.utf8Slice(".*coolfunctionname.*"); ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte); break; case "phone": pattern = Slices.utf8Slice("\\d{3}/\\d{3}/\\d{4}"); // 47: '/', 48-57: '0'-'9' ThreadLocalRandom.current().ints(47, 58).limit(sourceLength).forEach(sliceOutput::appendByte); break; case "literal": pattern = Slices.utf8Slice("literal"); // 97-122: 'a'-'z' ThreadLocalRandom.current().ints(97, 123).limit(sourceLength).forEach(sliceOutput::appendByte); break; default: throw new IllegalArgumentException("pattern: " + patternString + " not supported"); } joniPattern = joniRegexp(pattern); re2JPattern = re2JRegexp(pattern); source = sliceOutput.slice(); checkState(source.length() == sourceLength, "source.length=%s, sourceLength=%s", source.length(), sourceLength); }
Example #29
Source File: StringFunctions.java From presto with Apache License 2.0 | 5 votes |
@ScalarOperator(OperatorType.CAST) @LiteralParameters("x") @SqlType(CodePointsType.NAME) public static int[] castVarcharToCodePoints(@SqlType("varchar(x)") Slice slice) { return castToCodePoints(slice); }
Example #30
Source File: TestCastDependencies.java From presto with Apache License 2.0 | 5 votes |
@TypeParameter("V") @SqlType(StandardTypes.VARCHAR) public static Slice castAnyToVarchar( @CastDependency(fromType = "V", toType = StandardTypes.VARCHAR) MethodHandle cast, @SqlType("V") long value) { try { return (Slice) cast.invokeExact(value); } catch (Throwable t) { throwIfInstanceOf(t, Error.class); throwIfInstanceOf(t, PrestoException.class); throw new PrestoException(GENERIC_INTERNAL_ERROR, t); } }