Java Code Examples for io.airlift.slice.Slice#setBytes()

public static Slice padSpaces(Slice slice, int length)
{
    int textLength = countCodePoints(slice);

    if (textLength > length) {
        throw new IllegalArgumentException("pad length is smaller than slice length");
    }

    if (textLength == length) {
        return slice;
    }

    int bufferSize = slice.length() + length - textLength;
    Slice buffer = Slices.allocate(bufferSize);

    buffer.setBytes(0, slice);

    for (int i = slice.length(); i < bufferSize; ++i) {
        buffer.setByte(i, ' ');
    }

    return buffer;
}
 

@Description("Concatenates given character strings")
@ScalarFunction
@LiteralParameters({"x", "y", "u"})
@Constraint(variable = "u", expression = "x + y")
@SqlType("char(u)")
public static Slice concat(@LiteralParameter("x") Long x, @SqlType("char(x)") Slice left, @SqlType("char(y)") Slice right)
{
    int rightLength = right.length();
    if (rightLength == 0) {
        return left;
    }

    Slice paddedLeft = padSpaces(left, x.intValue());
    int leftLength = paddedLeft.length();

    Slice result = Slices.allocate(leftLength + rightLength);
    result.setBytes(0, paddedLeft);
    result.setBytes(leftLength, right);

    return result;
}
 

public static Slice pack(BigInteger unscaledValue, Slice result)
{
    pack(0, 0, false, result);
    byte[] bytes = unscaledValue.abs().toByteArray();

    if (bytes.length > UNSCALED_DECIMAL_128_SLICE_LENGTH
            || (bytes.length == UNSCALED_DECIMAL_128_SLICE_LENGTH && (bytes[0] & SIGN_BYTE_MASK) != 0)) {
        throwOverflowException();
    }

    // convert to little-endian order
    reverse(bytes);

    result.setBytes(0, bytes);
    if (unscaledValue.signum() < 0) {
        setNegative(result, true);
    }

    throwIfOverflows(result);

    return result;
}
 

private static List<Long> getSyncPositionsBruteForce(RcFileReader recordReader, File file)
{
    Slice slice = Slices.allocate(toIntExact(file.length()));
    try (InputStream in = new FileInputStream(file)) {
        slice.setBytes(0, in, slice.length());
    }
    catch (IOException e) {
        throw new UncheckedIOException(e);
    }

    List<Long> syncPositionsBruteForce = new ArrayList<>();
    Slice sync = Slices.allocate(SIZE_OF_INT + SIZE_OF_LONG + SIZE_OF_LONG);
    sync.setInt(0, -1);
    sync.setBytes(SIZE_OF_INT, recordReader.getSync());

    long syncPosition = 0;
    while (syncPosition >= 0) {
        syncPosition = slice.indexOf(sync, toIntExact(syncPosition));
        if (syncPosition > 0) {
            syncPositionsBruteForce.add(syncPosition);
            syncPosition++;
        }
    }
    return syncPositionsBruteForce;
}
 

private static Slice offsetHeapSlice(String value)
{
    Slice source = Slices.utf8Slice(value);
    Slice result = Slices.allocate(source.length() + 5);
    result.setBytes(2, source);
    return result.slice(2, source.length());
}
 

/**
 * Serializes the model using the following format
 * int: format version
 * byte[32]: SHA256 hash of all following data
 * int: id of algorithm
 * int: length of hyperparameters section
 * byte[]: hyperparameters (currently not used)
 * long: length of data section
 * byte[]: model data
 * <p>
 * note: all multibyte values are in little endian
 */
public static Slice serialize(Model model)
{
    requireNonNull(model, "model is null");
    Integer id = MODEL_SERIALIZATION_IDS.get(model.getClass());
    requireNonNull(id, "id is null");
    int size = HYPERPARAMETERS_OFFSET;

    // hyperparameters aren't implemented yet
    byte[] hyperparameters = new byte[0];
    size += hyperparameters.length;

    int dataLengthOffset = size;
    size += SIZE_OF_LONG;
    int dataOffset = size;
    byte[] data = model.getSerializedData();
    size += data.length;

    Slice slice = Slices.allocate(size);
    slice.setInt(VERSION_OFFSET, CURRENT_FORMAT_VERSION);
    slice.setInt(ALGORITHM_OFFSET, id);
    slice.setInt(HYPERPARAMETER_LENGTH_OFFSET, hyperparameters.length);
    slice.setBytes(HYPERPARAMETERS_OFFSET, hyperparameters);
    slice.setLong(dataLengthOffset, data.length);
    slice.setBytes(dataOffset, data);

    byte[] modelHash = Hashing.sha256().hashBytes(slice.getBytes(ALGORITHM_OFFSET, slice.length() - ALGORITHM_OFFSET)).asBytes();
    checkState(modelHash.length == 32, "sha256 hash code expected to be 32 bytes");
    slice.setBytes(HASH_OFFSET, modelHash);

    return slice;
}
 

@Override
public void decompress(Slice compressed, Slice uncompressed)
        throws RcFileCorruptionException
{
    checkState(!destroyed, "Codec has been destroyed");
    decompressor.reset();
    try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput(), decompressor)) {
        uncompressed.setBytes(0, decompressorStream, uncompressed.length());
    }
    catch (IndexOutOfBoundsException | IOException e) {
        throw new RcFileCorruptionException(e, "Compressed stream is truncated");
    }
}
 

@Override
public void decompress(Slice compressed, Slice uncompressed)
        throws RcFileCorruptionException
{
    try (CompressionInputStream decompressorStream = codec.createInputStream(compressed.getInput())) {
        uncompressed.setBytes(0, decompressorStream, uncompressed.length());
    }
    catch (IndexOutOfBoundsException | IOException e) {
        throw new RcFileCorruptionException(e, "Compressed stream is truncated");
    }
}
 

@VisibleForTesting
static Slice concatSlices(Slice slice1, Slice slice2)
{
    Slice slice = Slices.allocate(slice1.length() + slice2.length());
    slice.setBytes(0, slice1.getBytes());
    slice.setBytes(slice1.length(), slice2.getBytes());
    return slice;
}
 

private static Slice pad(Slice text, long targetLength, Slice padString, int paddingOffset)
{
    checkCondition(
            0 <= targetLength && targetLength <= Integer.MAX_VALUE,
            INVALID_FUNCTION_ARGUMENT,
            "Target length must be in the range [0.." + Integer.MAX_VALUE + "]");
    checkCondition(padString.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding string must not be empty");

    int textLength = countCodePoints(text);
    int resultLength = (int) targetLength;

    // if our target length is the same as our string then return our string
    if (textLength == resultLength) {
        return text;
    }

    // if our string is bigger than requested then truncate
    if (textLength > resultLength) {
        return SliceUtf8.substring(text, 0, resultLength);
    }

    // number of bytes in each code point
    int padStringLength = countCodePoints(padString);
    int[] padStringCounts = new int[padStringLength];
    for (int i = 0; i < padStringLength; ++i) {
        padStringCounts[i] = lengthOfCodePointSafe(padString, offsetOfCodePoint(padString, i));
    }

    // preallocate the result
    int bufferSize = text.length();
    for (int i = 0; i < resultLength - textLength; ++i) {
        bufferSize += padStringCounts[i % padStringLength];
    }

    Slice buffer = Slices.allocate(bufferSize);

    // fill in the existing string
    int countBytes = bufferSize - text.length();
    int startPointOfExistingText = (paddingOffset + countBytes) % bufferSize;
    buffer.setBytes(startPointOfExistingText, text);

    // assign the pad string while there's enough space for it
    int byteIndex = paddingOffset;
    for (int i = 0; i < countBytes / padString.length(); ++i) {
        buffer.setBytes(byteIndex, padString);
        byteIndex += padString.length();
    }

    // handle the tail: at most we assign padStringLength - 1 code points
    buffer.setBytes(byteIndex, padString.getBytes(0, paddingOffset + countBytes - byteIndex));
    return buffer;
}
 

private static Slice pad(Slice inputSlice, long targetLength, Slice padSlice, int paddingOffset)
{
    checkCondition(
            0 <= targetLength && targetLength <= Integer.MAX_VALUE,
            INVALID_FUNCTION_ARGUMENT,
            "Target length must be in the range [0.." + Integer.MAX_VALUE + "]");
    checkCondition(padSlice.length() > 0, INVALID_FUNCTION_ARGUMENT, "Padding bytes must not be empty");

    int inputLength = inputSlice.length();
    int resultLength = (int) targetLength;

    // if our target length is the same as our string then return our string
    if (inputLength == resultLength) {
        return inputSlice;
    }

    // if our string is bigger than requested then truncate
    if (inputLength > resultLength) {
        return inputSlice.slice(0, resultLength);
    }

    // preallocate the result
    Slice buffer = Slices.allocate(resultLength);

    // fill in the existing string
    int fillLength = resultLength - inputLength;
    int startPointOfExistingText = (paddingOffset + fillLength) % resultLength;
    buffer.setBytes(startPointOfExistingText, inputSlice);

    // assign the pad string while there's enough space for it
    int byteIndex = paddingOffset;
    for (int i = 0; i < fillLength / padSlice.length(); i++) {
        buffer.setBytes(byteIndex, padSlice);
        byteIndex += padSlice.length();
    }

    // handle the tail: at most we assign padStringLength - 1 code points
    buffer.setBytes(byteIndex, padSlice.getBytes(0, paddingOffset + fillLength - byteIndex));

    return buffer;
}
 

public OrcReader(OrcDataSource orcDataSource, MetadataReader metadataReader, DataSize maxMergeDistance, DataSize maxReadSize)
        throws IOException
{
    orcDataSource = wrapWithCacheIfTiny(requireNonNull(orcDataSource, "orcDataSource is null"), maxMergeDistance);
    this.orcDataSource = orcDataSource;
    this.metadataReader = requireNonNull(metadataReader, "metadataReader is null");
    this.maxMergeDistance = requireNonNull(maxMergeDistance, "maxMergeDistance is null");
    this.maxReadSize = requireNonNull(maxReadSize, "maxReadSize is null");

    //
    // Read the file tail:
    //
    // variable: Footer
    // variable: Metadata
    // variable: PostScript - contains length of footer and metadata
    // 3 bytes: file magic "ORC"
    // 1 byte: postScriptSize = PostScript + Magic

    // figure out the size of the file using the option or filesystem
    long size = orcDataSource.getSize();
    if (size <= 0) {
        throw new OrcCorruptionException("Malformed ORC file %s. Invalid file size %s", orcDataSource, size);
    }

    // Read the tail of the file
    byte[] buffer = new byte[toIntExact(min(size, EXPECTED_FOOTER_SIZE))];
    orcDataSource.readFully(size - buffer.length, buffer);

    // get length of PostScript - last byte of the file
    int postScriptSize = buffer[buffer.length - SIZE_OF_BYTE] & 0xff;

    // make sure this is an ORC file and not an RCFile or something else
    verifyOrcFooter(orcDataSource, postScriptSize, buffer);

    // decode the post script
    int postScriptOffset = buffer.length - SIZE_OF_BYTE - postScriptSize;
    PostScript postScript = metadataReader.readPostScript(buffer, postScriptOffset, postScriptSize);

    // verify this is a supported version
    checkOrcVersion(orcDataSource, postScript.getVersion());

    // check compression codec is supported
    this.compressionKind = postScript.getCompression();

    this.hiveWriterVersion = postScript.getHiveWriterVersion();
    this.bufferSize = toIntExact(postScript.getCompressionBlockSize());

    int footerSize = toIntExact(postScript.getFooterLength());
    int metadataSize = toIntExact(postScript.getMetadataLength());

    // check if extra bytes need to be read
    Slice completeFooterSlice;
    int completeFooterSize = footerSize + metadataSize + postScriptSize + SIZE_OF_BYTE;
    if (completeFooterSize > buffer.length) {
        // allocate a new buffer large enough for the complete footer
        byte[] newBuffer = new byte[completeFooterSize];
        completeFooterSlice = Slices.wrappedBuffer(newBuffer);

        // initial read was not large enough, so read missing section
        orcDataSource.readFully(size - completeFooterSize, newBuffer, 0, completeFooterSize - buffer.length);

        // copy already read bytes into the new buffer
        completeFooterSlice.setBytes(completeFooterSize - buffer.length, buffer);
    }
    else {
        // footer is already in the bytes in buffer, just adjust position, length
        completeFooterSlice = Slices.wrappedBuffer(buffer, buffer.length - completeFooterSize, completeFooterSize);
    }

    // read metadata
    Slice metadataSlice = completeFooterSlice.slice(0, metadataSize);
    try (InputStream metadataInputStream = new OrcInputStream(orcDataSource.toString(), metadataSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) {
        this.metadata = metadataReader.readMetadata(hiveWriterVersion, metadataInputStream);
    }

    // read footer
    Slice footerSlice = completeFooterSlice.slice(metadataSize, footerSize);
    try (InputStream footerInputStream = new OrcInputStream(orcDataSource.toString(), footerSlice.getInput(), compressionKind, bufferSize, new AggregatedMemoryContext())) {
        this.footer = metadataReader.readFooter(hiveWriterVersion, footerInputStream);
    }
}