Java Code Examples for io.airlift.slice.Slice#setBytes()
The following examples show how to use
io.airlift.slice.Slice#setBytes() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Chars.java From presto with Apache License 2.0 | 7 votes |
public static Slice padSpaces(Slice slice, int length) { int textLength = countCodePoints(slice); if (textLength > length) { throw new IllegalArgumentException("pad length is smaller than slice length"); } if (textLength == length) { return slice; } int bufferSize = slice.length() + length - textLength; Slice buffer = Slices.allocate(bufferSize); buffer.setBytes(0, slice); for (int i = slice.length(); i < bufferSize; ++i) { buffer.setByte(i, ' '); } return buffer; }
Example 2
Source File: StringFunctions.java From presto with Apache License 2.0 | 6 votes |
@Description("Concatenates given character strings") @ScalarFunction @LiteralParameters({"x", "y", "u"}) @Constraint(variable = "u", expression = "x + y") @SqlType("char(u)") public static Slice concat(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice left, @SqlType("char(y)") Slice right) { int rightLength = right.length(); if (rightLength == 0) { return left; } Slice paddedLeft = padSpaces(left, x.intValue()); int leftLength = paddedLeft.length(); Slice result = Slices.allocate(leftLength + rightLength); result.setBytes(0, paddedLeft); result.setBytes(leftLength, right); return result; }
Example 3
Source File: UnscaledDecimal128Arithmetic.java From presto with Apache License 2.0 | 6 votes |
public static Slice pack(BigInteger unscaledValue, Slice result) { pack(0, 0, false, result); byte[] bytes = unscaledValue.abs().toByteArray(); if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) { throwOverflowException(); } // convert to little-endian order reverse(bytes); result.setBytes(0, bytes); if (unscaledValue.signum() < 0) { setNegative(result, true); } throwIfOverflows(result); return result; }
Example 4
Source File: RcFileTester.java From presto with Apache License 2.0 | 6 votes |
private static List<Long> getSyncPositionsBruteForce(RcFileReader recordReader, File file) { Slice slice = Slices.allocate(toIntExact(file.length())); try (InputStream in = new FileInputStream(file)) { slice.setBytes(0, in, slice.length()); } catch (IOException e) { throw new UncheckedIOException(e); } List<Long> syncPositionsBruteForce = new ArrayList<>(); Slice sync = Slices.allocate(SIZE_OF_INT + SIZE_OF_LONG + SIZE_OF_LONG); sync.setInt(0, -1); sync.setBytes(SIZE_OF_INT, recordReader.getSync()); long syncPosition = 0; while (syncPosition >= 0) { syncPosition = slice.indexOf(sync, toIntExact(syncPosition)); if (syncPosition > 0) { syncPositionsBruteForce.add(syncPosition); syncPosition++; } } return syncPositionsBruteForce; }
Example 5
Source File: TestLikeFunctions.java From presto with Apache License 2.0 | 5 votes |
private static Slice offsetHeapSlice(String value) { Slice source = Slices.utf8Slice(value); Slice result = Slices.allocate(source.length() + 5); result.setBytes(2, source); return result.slice(2, source.length()); }
Example 6
Source File: ModelUtils.java From presto with Apache License 2.0 | 5 votes |
/** * Serializes the model using the following format * int: format version * byte[32]: SHA256 hash of all following data * int: id of algorithm * int: length of hyperparameters section * byte[]: hyperparameters (currently not used) * long: length of data section * byte[]: model data * <p> * note: all multibyte values are in little endian */ public static Slice serialize(Model model) { requireNonNull(model, "model is null"); Integer id = MODEL_SERIALIZATION_IDS.get(model.getClass()); requireNonNull(id, "id is null"); int size = HYPERPARAMETERS_OFFSET; // hyperparameters aren't implemented yet byte[] hyperparameters = new byte[0]; size += hyperparameters.length; int dataLengthOffset = size; size += SIZE_OF_LONG; int dataOffset = size; byte[] data = model.getSerializedData(); size += data.length; Slice slice = Slices.allocate(size); slice.setInt(VERSION_OFFSET, CURRENT_FORMAT_VERSION); slice.setInt(ALGORITHM_OFFSET, id); slice.setInt(HYPERPARAMETER_LENGTH_OFFSET, hyperparameters.length); slice.setBytes(HYPERPARAMETERS_OFFSET, hyperparameters); slice.setLong(dataLengthOffset, data.length); slice.setBytes(dataOffset, data); byte[] modelHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)).asBytes(); checkState(modelHash.length == 32, "sha256 hash code expected to be 32 bytes"); slice.setBytes(HASH_OFFSET, modelHash); return slice; }
Example 7
Source File: HadoopDecompressor.java From presto with Apache License 2.0 | 5 votes |
@Override public void decompress(Slice compressed, Slice uncompressed) throws RcFileCorruptionException { checkState(!destroyed, "Codec has been destroyed"); decompressor.reset(); try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput(), decompressor)) { uncompressed.setBytes(0, decompressorStream, uncompressed.length()); } catch (IndexOutOfBoundsException | IOException e) { throw new RcFileCorruptionException(e, "Compressed stream is truncated"); } }
Example 8
Source File: AircompressorDecompressor.java From presto with Apache License 2.0 | 5 votes |
@Override public void decompress(Slice compressed, Slice uncompressed) throws RcFileCorruptionException { try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput())) { uncompressed.setBytes(0, decompressorStream, uncompressed.length()); } catch (IndexOutOfBoundsException | IOException e) { throw new RcFileCorruptionException(e, "Compressed stream is truncated"); } }
Example 9
Source File: OrcMetadataReader.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
@VisibleForTesting static Slice concatSlices(Slice slice1, Slice slice2) { Slice slice = Slices.allocate(slice1.length() + slice2.length()); slice.setBytes(0, slice1.getBytes()); slice.setBytes(slice1.length(), slice2.getBytes()); return slice; }
Example 10
Source File: StringFunctions.java From presto with Apache License 2.0 | 4 votes |
private static Slice pad(Slice text, long targetLength, Slice padString, int paddingOffset) { checkCondition( 0 <= targetLength && targetLength <= Integer.MAX_VALUE, INVALID_FUNCTION_ARGUMENT, "Target length must be in the range [0.." + Integer.MAX_VALUE + "]"); checkCondition(padString.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding string must not be empty"); int textLength = countCodePoints(text); int resultLength = (int) targetLength; // if our target length is the same as our string then return our string if (textLength == resultLength) { return text; } // if our string is bigger than requested then truncate if (textLength > resultLength) { return SliceUtf8.substring(text, 0, resultLength); } // number of bytes in each code point int padStringLength = countCodePoints(padString); int[] padStringCounts = new int[padStringLength]; for (int i = 0; i < padStringLength; ++i) { padStringCounts[i] = lengthOfCodePointSafe(padString, offsetOfCodePoint(padString, i)); } // preallocate the result int bufferSize = text.length(); for (int i = 0; i < resultLength - textLength; ++i) { bufferSize += padStringCounts[i % padStringLength]; } Slice buffer = Slices.allocate(bufferSize); // fill in the existing string int countBytes = bufferSize - text.length(); int startPointOfExistingText = (paddingOffset + countBytes) % bufferSize; buffer.setBytes(startPointOfExistingText, text); // assign the pad string while there's enough space for it int byteIndex = paddingOffset; for (int i = 0; i < countBytes / padString.length(); ++i) { buffer.setBytes(byteIndex, padString); byteIndex += padString.length(); } // handle the tail: at most we assign padStringLength - 1 code points buffer.setBytes(byteIndex, padString.getBytes(0, paddingOffset + countBytes - byteIndex)); return buffer; }
Example 11
Source File: VarbinaryFunctions.java From presto with Apache License 2.0 | 4 votes |
private static Slice pad(Slice inputSlice, long targetLength, Slice padSlice, int paddingOffset) { checkCondition( 0 <= targetLength && targetLength <= Integer.MAX_VALUE, INVALID_FUNCTION_ARGUMENT, "Target length must be in the range [0.." + Integer.MAX_VALUE + "]"); checkCondition(padSlice.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding bytes must not be empty"); int inputLength = inputSlice.length(); int resultLength = (int) targetLength; // if our target length is the same as our string then return our string if (inputLength == resultLength) { return inputSlice; } // if our string is bigger than requested then truncate if (inputLength > resultLength) { return inputSlice.slice(0, resultLength); } // preallocate the result Slice buffer = Slices.allocate(resultLength); // fill in the existing string int fillLength = resultLength - inputLength; int startPointOfExistingText = (paddingOffset + fillLength) % resultLength; buffer.setBytes(startPointOfExistingText, inputSlice); // assign the pad string while there's enough space for it int byteIndex = paddingOffset; for (int i = 0; i < fillLength / padSlice.length(); i++) { buffer.setBytes(byteIndex, padSlice); byteIndex += padSlice.length(); } // handle the tail: at most we assign padStringLength - 1 code points buffer.setBytes(byteIndex, padSlice.getBytes(0, paddingOffset + fillLength - byteIndex)); return buffer; }
Example 12
Source File: OrcReader.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
public OrcReader(OrcDataSource orcDataSource, MetadataReader metadataReader, DataSize maxMergeDistance, DataSize maxReadSize) throws IOException { orcDataSource = wrapWithCacheIfTiny(requireNonNull(orcDataSource, "orcDataSource is null"), maxMergeDistance); this.orcDataSource = orcDataSource; this.metadataReader = requireNonNull(metadataReader, "metadataReader is null"); this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null"); this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null"); // // Read the file tail: // // variable: Footer // variable: Metadata // variable: PostScript - contains length of footer and metadata // 3 bytes: file magic "ORC" // 1 byte: postScriptSize = PostScript + Magic // figure out the size of the file using the option or filesystem long size = orcDataSource.getSize(); if (size <= 0) { throw new OrcCorruptionException("Malformed ORC file %s. Invalid file size %s", orcDataSource, size); } // Read the tail of the file byte[] buffer = new byte[toIntExact(min(size, EXPECTED_FOOTER_SIZE))]; orcDataSource.readFully(size - buffer.length, buffer); // get length of PostScript - last byte of the file int postScriptSize = buffer[buffer.length - SIZE_OF_BYTE] & 0xff; // make sure this is an ORC file and not an RCFile or something else verifyOrcFooter(orcDataSource, postScriptSize, buffer); // decode the post script int postScriptOffset = buffer.length - SIZE_OF_BYTE - postScriptSize; PostScript postScript = metadataReader.readPostScript(buffer, postScriptOffset, postScriptSize); // verify this is a supported version checkOrcVersion(orcDataSource, postScript.getVersion()); // check compression codec is supported this.compressionKind = postScript.getCompression(); this.hiveWriterVersion = postScript.getHiveWriterVersion(); this.bufferSize = toIntExact(postScript.getCompressionBlockSize()); int footerSize = toIntExact(postScript.getFooterLength()); int metadataSize = toIntExact(postScript.getMetadataLength()); // check if extra bytes need to be read Slice completeFooterSlice; int completeFooterSize = footerSize + metadataSize + postScriptSize + SIZE_OF_BYTE; if (completeFooterSize > buffer.length) { // allocate a new buffer large enough for the complete footer byte[] newBuffer = new byte[completeFooterSize]; completeFooterSlice = Slices.wrappedBuffer(newBuffer); // initial read was not large enough, so read missing section orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length); // copy already read bytes into the new buffer completeFooterSlice.setBytes(completeFooterSize - buffer.length, buffer); } else { // footer is already in the bytes in buffer, just adjust position, length completeFooterSlice = Slices.wrappedBuffer(buffer, buffer.length - completeFooterSize, completeFooterSize); } // read metadata Slice metadataSlice = completeFooterSlice.slice(0, metadataSize); try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.toString(), metadataSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) { this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream); } // read footer Slice footerSlice = completeFooterSlice.slice(metadataSize, footerSize); try (InputStream footerInputStream = new OrcInputStream(orcDataSource.toString(), footerSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) { this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream); } }