Java Code Examples for io.airlift.slice.Slice#setBytes()

The following examples show how to use io.airlift.slice.Slice#setBytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Chars.java    From presto with Apache License 2.0 7 votes vote down vote up
public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 
Example 2
Source File: StringFunctions.java    From presto with Apache License 2.0 6 votes vote down vote up
@Description("Concatenates given character strings")
@ScalarFunction
@LiteralParameters({"x", "y", "u"})
@Constraint(variable = "u", expression = "x + y")
@SqlType("char(u)")
public static Slice concat(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice left, @SqlType("char(y)") Slice right)
{
    int rightLength = right.length();
    if (rightLength == 0) {
        return left;
    }

    Slice paddedLeft = padSpaces(left, x.intValue());
    int leftLength = paddedLeft.length();

    Slice result = Slices.allocate(leftLength + rightLength);
    result.setBytes(0, paddedLeft);
    result.setBytes(leftLength, right);

    return result;
}
 
Example 3
Source File: UnscaledDecimal128Arithmetic.java    From presto with Apache License 2.0 6 votes vote down vote up
public static Slice pack(BigInteger unscaledValue, Slice result)
{
    pack(0, 0, false, result);
    byte[] bytes = unscaledValue.abs().toByteArray();

    if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH
            || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) {
        throwOverflowException();
    }

    // convert to little-endian order
    reverse(bytes);

    result.setBytes(0, bytes);
    if (unscaledValue.signum() < 0) {
        setNegative(result, true);
    }

    throwIfOverflows(result);

    return result;
}
 
Example 4
Source File: RcFileTester.java    From presto with Apache License 2.0 6 votes vote down vote up
private static List<Long> getSyncPositionsBruteForce(RcFileReader recordReader, File file)
{
    Slice slice = Slices.allocate(toIntExact(file.length()));
    try (InputStream in = new FileInputStream(file)) {
        slice.setBytes(0, in, slice.length());
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }

    List<Long> syncPositionsBruteForce = new ArrayList<>();
    Slice sync = Slices.allocate(SIZE_OF_INT + SIZE_OF_LONG + SIZE_OF_LONG);
    sync.setInt(0, -1);
    sync.setBytes(SIZE_OF_INT, recordReader.getSync());

    long syncPosition = 0;
    while (syncPosition >= 0) {
        syncPosition = slice.indexOf(sync, toIntExact(syncPosition));
        if (syncPosition > 0) {
            syncPositionsBruteForce.add(syncPosition);
            syncPosition++;
        }
    }
    return syncPositionsBruteForce;
}
 
Example 5
Source File: TestLikeFunctions.java    From presto with Apache License 2.0 5 votes vote down vote up
private static Slice offsetHeapSlice(String value)
{
    Slice source = Slices.utf8Slice(value);
    Slice result = Slices.allocate(source.length() + 5);
    result.setBytes(2, source);
    return result.slice(2, source.length());
}
 
Example 6
Source File: ModelUtils.java    From presto with Apache License 2.0 5 votes vote down vote up
/**
 * Serializes the model using the following format
 * int: format version
 * byte[32]: SHA256 hash of all following data
 * int: id of algorithm
 * int: length of hyperparameters section
 * byte[]: hyperparameters (currently not used)
 * long: length of data section
 * byte[]: model data
 * <p>
 * note: all multibyte values are in little endian
 */
public static Slice serialize(Model model)
{
    requireNonNull(model, "model is null");
    Integer id = MODEL_SERIALIZATION_IDS.get(model.getClass());
    requireNonNull(id, "id is null");
    int size = HYPERPARAMETERS_OFFSET;

    // hyperparameters aren't implemented yet
    byte[] hyperparameters = new byte[0];
    size += hyperparameters.length;

    int dataLengthOffset = size;
    size += SIZE_OF_LONG;
    int dataOffset = size;
    byte[] data = model.getSerializedData();
    size += data.length;

    Slice slice = Slices.allocate(size);
    slice.setInt(VERSION_OFFSET, CURRENT_FORMAT_VERSION);
    slice.setInt(ALGORITHM_OFFSET, id);
    slice.setInt(HYPERPARAMETER_LENGTH_OFFSET, hyperparameters.length);
    slice.setBytes(HYPERPARAMETERS_OFFSET, hyperparameters);
    slice.setLong(dataLengthOffset, data.length);
    slice.setBytes(dataOffset, data);

    byte[] modelHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)).asBytes();
    checkState(modelHash.length == 32, "sha256 hash code expected to be 32 bytes");
    slice.setBytes(HASH_OFFSET, modelHash);

    return slice;
}
 
Example 7
Source File: HadoopDecompressor.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public void decompress(Slice compressed, Slice uncompressed)
        throws RcFileCorruptionException
{
    checkState(!destroyed, "Codec has been destroyed");
    decompressor.reset();
    try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput(), decompressor)) {
        uncompressed.setBytes(0, decompressorStream, uncompressed.length());
    }
    catch (IndexOutOfBoundsException | IOException e) {
        throw new RcFileCorruptionException(e, "Compressed stream is truncated");
    }
}
 
Example 8
Source File: AircompressorDecompressor.java    From presto with Apache License 2.0 5 votes vote down vote up
@Override
public void decompress(Slice compressed, Slice uncompressed)
        throws RcFileCorruptionException
{
    try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput())) {
        uncompressed.setBytes(0, decompressorStream, uncompressed.length());
    }
    catch (IndexOutOfBoundsException | IOException e) {
        throw new RcFileCorruptionException(e, "Compressed stream is truncated");
    }
}
 
Example 9
Source File: OrcMetadataReader.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@VisibleForTesting
static Slice concatSlices(Slice slice1, Slice slice2)
{
    Slice slice = Slices.allocate(slice1.length() + slice2.length());
    slice.setBytes(0, slice1.getBytes());
    slice.setBytes(slice1.length(), slice2.getBytes());
    return slice;
}
 
Example 10
Source File: StringFunctions.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Slice pad(Slice text, long targetLength, Slice padString, int paddingOffset)
{
    checkCondition(
            0 <= targetLength && targetLength <= Integer.MAX_VALUE,
            INVALID_FUNCTION_ARGUMENT,
            "Target length must be in the range [0.." + Integer.MAX_VALUE + "]");
    checkCondition(padString.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding string must not be empty");

    int textLength = countCodePoints(text);
    int resultLength = (int) targetLength;

    // if our target length is the same as our string then return our string
    if (textLength == resultLength) {
        return text;
    }

    // if our string is bigger than requested then truncate
    if (textLength > resultLength) {
        return SliceUtf8.substring(text, 0, resultLength);
    }

    // number of bytes in each code point
    int padStringLength = countCodePoints(padString);
    int[] padStringCounts = new int[padStringLength];
    for (int i = 0; i < padStringLength; ++i) {
        padStringCounts[i] = lengthOfCodePointSafe(padString, offsetOfCodePoint(padString, i));
    }

    // preallocate the result
    int bufferSize = text.length();
    for (int i = 0; i < resultLength - textLength; ++i) {
        bufferSize += padStringCounts[i % padStringLength];
    }

    Slice buffer = Slices.allocate(bufferSize);

    // fill in the existing string
    int countBytes = bufferSize - text.length();
    int startPointOfExistingText = (paddingOffset + countBytes) % bufferSize;
    buffer.setBytes(startPointOfExistingText, text);

    // assign the pad string while there's enough space for it
    int byteIndex = paddingOffset;
    for (int i = 0; i < countBytes / padString.length(); ++i) {
        buffer.setBytes(byteIndex, padString);
        byteIndex += padString.length();
    }

    // handle the tail: at most we assign padStringLength - 1 code points
    buffer.setBytes(byteIndex, padString.getBytes(0, paddingOffset + countBytes - byteIndex));
    return buffer;
}
 
Example 11
Source File: VarbinaryFunctions.java    From presto with Apache License 2.0 4 votes vote down vote up
private static Slice pad(Slice inputSlice, long targetLength, Slice padSlice, int paddingOffset)
{
    checkCondition(
            0 <= targetLength && targetLength <= Integer.MAX_VALUE,
            INVALID_FUNCTION_ARGUMENT,
            "Target length must be in the range [0.." + Integer.MAX_VALUE + "]");
    checkCondition(padSlice.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding bytes must not be empty");

    int inputLength = inputSlice.length();
    int resultLength = (int) targetLength;

    // if our target length is the same as our string then return our string
    if (inputLength == resultLength) {
        return inputSlice;
    }

    // if our string is bigger than requested then truncate
    if (inputLength > resultLength) {
        return inputSlice.slice(0, resultLength);
    }

    // preallocate the result
    Slice buffer = Slices.allocate(resultLength);

    // fill in the existing string
    int fillLength = resultLength - inputLength;
    int startPointOfExistingText = (paddingOffset + fillLength) % resultLength;
    buffer.setBytes(startPointOfExistingText, inputSlice);

    // assign the pad string while there's enough space for it
    int byteIndex = paddingOffset;
    for (int i = 0; i < fillLength / padSlice.length(); i++) {
        buffer.setBytes(byteIndex, padSlice);
        byteIndex += padSlice.length();
    }

    // handle the tail: at most we assign padStringLength - 1 code points
    buffer.setBytes(byteIndex, padSlice.getBytes(0, paddingOffset + fillLength - byteIndex));

    return buffer;
}
 
Example 12
Source File: OrcReader.java    From spliceengine with GNU Affero General Public License v3.0 4 votes vote down vote up
public OrcReader(OrcDataSource orcDataSource, MetadataReader metadataReader, DataSize maxMergeDistance, DataSize maxReadSize)
        throws IOException
{
    orcDataSource = wrapWithCacheIfTiny(requireNonNull(orcDataSource, "orcDataSource is null"), maxMergeDistance);
    this.orcDataSource = orcDataSource;
    this.metadataReader = requireNonNull(metadataReader, "metadataReader is null");
    this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null");
    this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null");

    //
    // Read the file tail:
    //
    // variable: Footer
    // variable: Metadata
    // variable: PostScript - contains length of footer and metadata
    // 3 bytes: file magic "ORC"
    // 1 byte: postScriptSize = PostScript + Magic

    // figure out the size of the file using the option or filesystem
    long size = orcDataSource.getSize();
    if (size <= 0) {
        throw new OrcCorruptionException("Malformed ORC file %s. Invalid file size %s", orcDataSource, size);
    }

    // Read the tail of the file
    byte[] buffer = new byte[toIntExact(min(size, EXPECTED_FOOTER_SIZE))];
    orcDataSource.readFully(size - buffer.length, buffer);

    // get length of PostScript - last byte of the file
    int postScriptSize = buffer[buffer.length - SIZE_OF_BYTE] & 0xff;

    // make sure this is an ORC file and not an RCFile or something else
    verifyOrcFooter(orcDataSource, postScriptSize, buffer);

    // decode the post script
    int postScriptOffset = buffer.length - SIZE_OF_BYTE - postScriptSize;
    PostScript postScript = metadataReader.readPostScript(buffer, postScriptOffset, postScriptSize);

    // verify this is a supported version
    checkOrcVersion(orcDataSource, postScript.getVersion());

    // check compression codec is supported
    this.compressionKind = postScript.getCompression();

    this.hiveWriterVersion = postScript.getHiveWriterVersion();
    this.bufferSize = toIntExact(postScript.getCompressionBlockSize());

    int footerSize = toIntExact(postScript.getFooterLength());
    int metadataSize = toIntExact(postScript.getMetadataLength());

    // check if extra bytes need to be read
    Slice completeFooterSlice;
    int completeFooterSize = footerSize + metadataSize + postScriptSize + SIZE_OF_BYTE;
    if (completeFooterSize > buffer.length) {
        // allocate a new buffer large enough for the complete footer
        byte[] newBuffer = new byte[completeFooterSize];
        completeFooterSlice = Slices.wrappedBuffer(newBuffer);

        // initial read was not large enough, so read missing section
        orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length);

        // copy already read bytes into the new buffer
        completeFooterSlice.setBytes(completeFooterSize - buffer.length, buffer);
    }
    else {
        // footer is already in the bytes in buffer, just adjust position, length
        completeFooterSlice = Slices.wrappedBuffer(buffer, buffer.length - completeFooterSize, completeFooterSize);
    }

    // read metadata
    Slice metadataSlice = completeFooterSlice.slice(0, metadataSize);
    try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.toString(), metadataSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) {
        this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream);
    }

    // read footer
    Slice footerSlice = completeFooterSlice.slice(metadataSize, footerSize);
    try (InputStream footerInputStream = new OrcInputStream(orcDataSource.toString(), footerSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) {
        this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream);
    }
}