Java Code Examples for net.openhft.chronicle.bytes.Bytes#writePosition()

@Test
public void testHash() {
    if ((testCounter % 250) == 0) {
        long newTime = System.currentTimeMillis();
        System.out.println("Executing test number " + testCounter + " for data size " + size + " time since last log "
                + String.format("%.2f", ((newTime - timePassed) / 1000.0)) + " sec(s)");
        timePassed = newTime;
    }
    testCounter++;
    Bytes<?> bytesMessage = testDataBytes;
    bytesMessage.readPositionRemaining(0, size);
    Bytes<?> sha256Actual = hash256Bytes.get();
    sha256Actual.writePosition(0);
    SHA2.appendSha256(sha256Actual, bytesMessage);
    sha256Actual.readPosition(0);

    Bytes<?> sha256Expected = bft.fromHex(sha256);
    sha256Expected.readPosition(0);

    assertEquals(sha256Expected.toHexString(SHA2.HASH_SHA256_BYTES), sha256Actual.toHexString(SHA2.HASH_SHA256_BYTES));
    sha256Expected.releaseLast();
}
 

private void removeEOF(Path path) throws IOException {
    long blockSize = 64 << 10;
    long chunkSize = OS.pageAlign(blockSize);
    long overlapSize = OS.pageAlign(blockSize / 4);
    final MappedBytes mappedBytes = MappedBytes.mappedBytes(path.toFile(), chunkSize, overlapSize, false);
    mappedBytes.reserve(test);
    try {
        final Wire wire = WireType.BINARY_LIGHT.apply(mappedBytes);
        final Bytes<?> bytes = wire.bytes();
        bytes.readLimitToCapacity();
        bytes.readSkip(4);
        // move past header
        try (final SingleChronicleQueueStore qs = loadStore(wire)) {
            assertNotNull(qs);
            long l = qs.writePosition();
            long len = Wires.lengthOf(bytes.readVolatileInt(l));
            long eofOffset = l + len + 4L;
            bytes.writePosition(eofOffset);
            bytes.writeInt(0);
        }
    } finally {
        mappedBytes.release(test);
    }
}
 

private static void logToStandardOutMessageReceivedInERROR(@NotNull final Wire wire) {
    @NotNull final Bytes<?> bytes = wire.bytes();

    final long position = bytes.writePosition();
    final long limit = bytes.writeLimit();
    try {
        try {

            LOG.info("\nreceives IN ERROR:\n" +
                    "```yaml\n" +
                    Wires.fromSizePrefixedBlobs(wire) +
                    "```\n");
            YamlLogging.title = "";
            YamlLogging.writeMessage("");

        } catch (Exception e) {
            String x = Bytes.toString(bytes);
            Jvm.warn().on(TcpChannelHub.class, x, e);
        }
    } finally {
        bytes.writeLimit(limit);
        bytes.writePosition(position);
    }
}
 

/**
 * the following steps
 * <p>
 * (1) write to a queue
 * (2) force and end for file marker
 * (3) write to the queue again, this will cause it to be written to tomorrows .cq4 file
 * (4) delete todays queue files, that was created in step (1)
 * (5) create a new instance of chronicle queue ( same directory ) as step 1
 * (6) create a tailer toEnd
 * (6) write to this queue created in (5)
 * (7) when you now try to read from this queue you will not be able to read back what you have just written in (6)
 */
@Test
public void test() {
    File tmpDir = getTmpDir();
    File[] files;
    try (ChronicleQueue writeQ = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptAppender appender = writeQ.acquireAppender();
        long wp;
        Wire wire;

        try (DocumentContext dc = appender.writingDocument()) {
            wire = dc.wire();
            wire.write().text("hello world");
            Bytes<?> bytes = wire.bytes();
            wp = bytes.writePosition();
        }

        File dir = new File(appender.queue().fileAbsolutePath());
        files = dir.listFiles(pathname -> pathname.getAbsolutePath().endsWith(".cq4"));

        wire.bytes().writeInt(wp, Wires.END_OF_DATA);
        appender.writeText("hello world  2");
    }

    Assert.assertEquals(files.length, 1);
    File file = files[0];
    file.delete();

    try (ChronicleQueue q = ChronicleQueue.singleBuilder(tmpDir).build()) {
        ExcerptTailer excerptTailer = q.createTailer().toEnd();
        q.acquireAppender()
                .writeText(EXPECTED);
        Assert.assertEquals(EXPECTED, excerptTailer.readText());
    }
}
 

static Bytes<?> fromHex(int padding, String s) {
    byte[] byteArr = DatatypeConverter.parseHexBinary(s);
    Bytes<?> bytes = Bytes.allocateDirect(padding + byteArr.length);
    if (padding > 0) {
        bytes.zeroOut(0, padding);
        bytes.writePosition(padding);
    }
    bytes.write(byteArr);
    return bytes;
}
 

@Test
public void Appender() {
    if (!assert_from_cpp())
        return;

    String tmp = "/dev/shm/RawAccessJtoC";
    System.out.println(tmp); // so C++ knows this ran rather than skipped

    try (ChronicleQueue cq = SingleChronicleQueueBuilder.binary(tmp).build()) {

        ExcerptAppender appender = cq.acquireAppender();

        for (int i = 0; i < COUNT; ++i) {
            try (DocumentContext dc = appender.writingDocument()) {

                Bytes bytes = dc.wire().bytes();

                // will contain the size of the blob
                long start = bytes.writePosition();
                bytes.writeSkip(RAW_SIZE_PREFIX);

                {
                    bytes.writeByte((byte) 0xab);
                    bytes.writeShort((short) 12);
                    bytes.writeInt(123);
                    bytes.writeLong(123456789L);
                    bytes.writeFloat(1.234f);
                    bytes.writeDouble(123.456);
                    bytes.writeChar('a');
                    bytes.write8bit("Hello World");
                }

                long end = bytes.writePosition();
                bytes.writeInt(start, (int) (end - start - RAW_SIZE_PREFIX));
            }
        }
    }
}
 

/**
 * Creates a new Excerpt containing and index which will be 1L << 17L bytes long, This method is used for creating both the primary and secondary
 * indexes. Chronicle Queue uses a root primary index ( each entry in the primary index points to a unique a secondary index. The secondary index
 * only records the addressForRead of every 64th except, the except are linearly scanned from there on.  )
 *
 * @param wire the current wire
 * @return the addressForRead of the Excerpt containing the usable index, just after the header
 */
long newIndex(@NotNull WireOut wire, boolean index2index) throws StreamCorruptedException {
    long writePosition = this.writePosition.getVolatileValue();
    Bytes<?> bytes = wire.bytes();
    bytes.writePosition(writePosition);

    long position = wire.enterHeader(indexCount * 8 + 128);

    WriteMarshallable writer = index2index ? index2IndexTemplate : indexTemplate;
    writer.writeMarshallable(wire);
    wire.updateHeader(position, true, 0);

    return position;
}
 

private static void writeMessage(Wire wire, int messageSize) {
    Bytes<?> bytes = wire.bytes();
    long wp = bytes.writePosition();
    long addr = bytes.addressForWrite(wp);
    Memory memory = OS.memory();
    for (int i = 0; i < messageSize; i += 16) {
        memory.writeLong(addr + i, 0L);
        memory.writeLong(addr + i + 8, 0L);
    }

    bytes.writeSkip(messageSize);
    bytes.writeLong(wp, System.nanoTime());
}
 

public static int size(final BytesMarshallable message) {
    final Bytes<ByteBuffer> buffer = Bytes.elasticByteBuffer();
    try {
        message.writeMarshallable(buffer);
        return (int) buffer.writePosition();
    } finally {
        buffer.releaseLast();
    }
}
 

public void writeNewEntry(long pos, Data<?> key) {
    initPos(pos);
    initKeySize(key.size());
    Bytes segmentBytes = s.segmentBytesForWrite();
    segmentBytes.writePosition(keySizeOffset);
    hh.h().keySizeMarshaller.writeSize(segmentBytes, keySize);
    initKeyOffset(segmentBytes.writePosition());
    key.writeTo(s.segmentBS, keyOffset);
}
 

void initValueSize(long valueSize) {
    this.valueSize = valueSize;
    Bytes segmentBytes = s.segmentBytesForWrite();
    segmentBytes.writePosition(valueSizeOffset);
    mh.m().valueSizeMarshaller.writeSize(segmentBytes, valueSize);
    long currentPosition = segmentBytes.writePosition();
    long currentAddr = segmentBytes.addressForRead(currentPosition);
    long skip = alignAddr(currentAddr, mh.m().alignment) - currentAddr;
    if (skip > 0)
        segmentBytes.writeSkip(skip);
    valueOffset = segmentBytes.writePosition();
}
 

void updateReplicationState(byte identifier, long timestamp) {
    initDelayedUpdateChecksum(true);
    Bytes segmentBytes = s.segmentBytesForWrite();
    segmentBytes.writePosition(replicationBytesOffset);
    segmentBytes.writeLong(timestamp);
    segmentBytes.writeByte(identifier);
}
 

/**
 * @return ByteBuffer, with self bootstrapping header in [position, limit) range
 */
private static <K, V> ByteBuffer writeHeader(
        FileChannel fileChannel, VanillaChronicleMap<K, V, ?> map) throws IOException {
    ByteBuffer headerBuffer = ByteBuffer.allocate(
            SELF_BOOTSTRAPPING_HEADER_OFFSET + MAX_BOOTSTRAPPING_HEADER_SIZE);
    headerBuffer.order(LITTLE_ENDIAN);

    Bytes<ByteBuffer> headerBytes = Bytes.wrapForWrite(headerBuffer);
    headerBytes.writePosition(SELF_BOOTSTRAPPING_HEADER_OFFSET);
    Wire wire = new TextWire(headerBytes);
    wire.getValueOut().typedMarshallable(map);

    int headerLimit = (int) headerBytes.writePosition();
    int headerSize = headerLimit - SELF_BOOTSTRAPPING_HEADER_OFFSET;
    // First set readiness bit to READY, to compute checksum correctly
    //noinspection PointlessBitwiseExpression
    headerBuffer.putInt(SIZE_WORD_OFFSET, READY | DATA | headerSize);

    long checksum = headerChecksum(headerBuffer, headerSize);
    headerBuffer.putLong(HEADER_OFFSET, checksum);

    // Set readiness bit to NOT_COMPLETE, because the Chronicle Map instance is not actually
    // ready yet
    //noinspection PointlessBitwiseExpression
    headerBuffer.putInt(SIZE_WORD_OFFSET, NOT_COMPLETE | DATA | headerSize);

    // Write the size-prefixed blob to the file
    headerBuffer.position(0).limit(headerLimit);
    writeFully(fileChannel, 0, headerBuffer);

    headerBuffer.position(SELF_BOOTSTRAPPING_HEADER_OFFSET);
    return headerBuffer;
}
 

/**
 * blocks indefinitely until the number of expected bytes is received
 *
 * @param wire          the wire that the data will be written into, this wire must contain an underlying ByteBuffer
 * @param numberOfBytes the size of the data to read
 * @throws IOException if anything bad happens to the socket connection
 */
private void blockingRead(@NotNull final WireIn wire, final long numberOfBytes) throws IOException {

    @NotNull final Bytes<?> bytes = wire.bytes();
    bytes.ensureCapacity(bytes.writePosition() + numberOfBytes);

    @NotNull final ByteBuffer buffer = (ByteBuffer) bytes.underlyingObject();
    final int start = (int) bytes.writePosition();
    //noinspection ConstantConditions
    buffer.position(start);

    buffer.limit((int) (start + numberOfBytes));
    readBuffer(buffer);
    bytes.readLimit(buffer.position());
}
 

private static void logToStandardOutMessageReceived(@NotNull final Wire wire) {
    @NotNull final Bytes<?> bytes = wire.bytes();

    if (!YamlLogging.showClientReads())
        return;

    final long position = bytes.writePosition();
    final long limit = bytes.writeLimit();
    try {
        try {

            //         LOG.info("Bytes.toString(bytes)=" + Bytes.toString(bytes));
            LOG.info("\nreceives:\n" +
                    "```yaml\n" +
                    Wires.fromSizePrefixedBlobs(wire) +
                    "```\n");
            YamlLogging.title = "";
            YamlLogging.writeMessage("");

        } catch (Exception e) {
            Jvm.warn().on(TcpChannelHub.class, Bytes.toString(bytes), e);
        }
    } finally {
        bytes.writeLimit(limit);
        bytes.writePosition(position);
    }
}
 

@Test
public void signAndVerify() {
    Bytes privateKeyBuffer = null;
    Bytes secretKeyBuffer = null;
    Bytes privateOrSecret = bft.fromHex(privateOrSecretKey);
    if (privateOrSecret.readRemaining() == Ed25519.SECRET_KEY_LENGTH) {
        secretKeyBuffer = privateOrSecret;
    } else {
        privateKeyBuffer = privateOrSecret;
    }

    Bytes publicKeyBuffer = bft.fromHex(publicKey);
    if (secretKeyBuffer == null) {
        secretKeyBuffer = bft.bytesWithZeros(Ed25519.SECRET_KEY_LENGTH);
        Bytes tmpPublicKeyBuffer = bft.bytesWithZeros(Ed25519.PUBLIC_KEY_LENGTH);
        Ed25519.privateToPublicAndSecret(tmpPublicKeyBuffer, secretKeyBuffer, privateKeyBuffer);
        assertEquals(publicKeyBuffer.toHexString(), tmpPublicKeyBuffer.toHexString());
    }
    Bytes messageBuffer = bft.fromHex(message);
    Bytes signExpectedBuffer;
    if (signExpected.length() == 128) {
        signExpectedBuffer = Bytes.wrapForRead(DatatypeConverter.parseHexBinary(signExpected + message));
    } else {
        signExpectedBuffer = Bytes.wrapForRead(DatatypeConverter.parseHexBinary(signExpected));
    }
    Bytes signedMsgBuffer = bft.fromHex(Ed25519.SIGNATURE_LENGTH, message);
    signedMsgBuffer.writePosition(0);
    Ed25519.sign(signedMsgBuffer, messageBuffer, secretKeyBuffer);
    assertEquals(signExpectedBuffer.toHexString(), signedMsgBuffer.toHexString());
    signedMsgBuffer.readPosition(0);
    publicKeyBuffer.readPositionRemaining(0, Ed25519.PUBLIC_KEY_LENGTH);
    assertTrue(Ed25519.verify(signedMsgBuffer, publicKeyBuffer));
}
 

Bytes fromHex(int padding, String s) {
    byte[] byteArr = DatatypeConverter.parseHexBinary(s);
    Bytes bytes = bytesWithZeros(padding + byteArr.length);
    if (padding > 0) {
        bytes.zeroOut(0, padding);
        bytes.writePosition(padding);
    }
    bytes.write(byteArr);
    return bytes;
}
 

@Test
public void testHash() {
    Bytes<?> bytesMessage = bft.fromHex(data);
    bytesMessage.readPosition(0);
    bytesMessage.readPositionRemaining(0, size);

    Bytes<?> actualSha256 = hash256Bytes.get();
    actualSha256.writePosition(0);
    SHA2.appendSha256(actualSha256, bytesMessage);
    actualSha256.readPosition(0);
    Bytes<?> expectedSha256 = bft.fromHex(sha256);
    actualSha256.readPosition(0);
    assertEquals(expectedSha256.toHexString(), actualSha256.toHexString());
    expectedSha256.releaseLast();

    bytesMessage.readPositionRemaining(0, size);
    Bytes<?> actualSha512 = hash512Bytes.get();
    actualSha512.writePosition(0);
    SHA2.appendSha512(actualSha512, bytesMessage);
    actualSha512.readPosition(0);
    Bytes<?> expectedSha512 = bft.fromHex(sha512);
    actualSha512.readPosition(0);
    assertEquals(expectedSha512.toHexString(), actualSha512.toHexString());
    expectedSha512.releaseLast();

    bytesMessage.releaseLast();
}
 

@Override
public void write(Bytes out, @NotNull CharSequence cs) {
    // Write the actual cs length for accurate StringBuilder.ensureCapacity() while reading
    out.writeStopBit(cs.length());
    long encodedSizePos = out.writePosition();
    out.writeSkip(4);
    charsetEncoder.reset();
    inputBuffer.clear();
    outputBuffer.clear();
    int csPos = 0;
    boolean endOfInput = false;
    // this loop inspired by the CharsetEncoder.encode(CharBuffer) implementation
    while (true) {
        if (!endOfInput) {
            int nextCsPos = Math.min(csPos + inputBuffer.remaining(), cs.length());
            append(inputBuffer, cs, csPos, nextCsPos);
            inputBuffer.flip();
            endOfInput = nextCsPos == cs.length();
            csPos = nextCsPos;
        }

        CoderResult cr = inputBuffer.hasRemaining() ?
                charsetEncoder.encode(inputBuffer, outputBuffer, endOfInput) :
                CoderResult.UNDERFLOW;

        if (cr.isUnderflow() && endOfInput)
            cr = charsetEncoder.flush(outputBuffer);

        if (cr.isUnderflow()) {
            if (endOfInput) {
                break;
            } else {
                inputBuffer.compact();
                continue;
            }
        }

        if (cr.isOverflow()) {
            outputBuffer.flip();
            writeOutputBuffer(out);
            outputBuffer.clear();
            continue;
        }

        try {
            cr.throwException();
        } catch (CharacterCodingException e) {
            throw new IORuntimeException(e);
        }
    }
    outputBuffer.flip();
    writeOutputBuffer(out);

    out.writeInt(encodedSizePos, (int) (out.writePosition() - encodedSizePos - 4));
}
 

public static void main(String[] args) {
    final String SIGN_PRIVATE = "b18e1d0045995ec3d010c387ccfeb984d783af8fbb0f40fa7db126d889f6dadd";

    Bytes<?> publicKey = Bytes.allocateDirect(32);
    Bytes<?> secretKey = Bytes.allocateDirect(64);
    BytesStore<?, ?> privateKey = fromHex(SIGN_PRIVATE);
    Ed25519.privateToPublicAndSecret(publicKey, secretKey, privateKey);
    assertEquals(32, publicKey.readRemaining());
    assertEquals(64, secretKey.readRemaining());

    ThreadLocal<Bytes<?>> sigAndMsg = ThreadLocal.withInitial(() -> Bytes.allocateDirect(64 + 64));
    ThreadLocal<Bytes<?>> sigAndMsg2 = ThreadLocal.withInitial(() -> Bytes.allocateDirect(64 + 64));

    int procs = Runtime.getRuntime().availableProcessors();
    for (int t = 0; t < 10; t++) {
        int runs = procs * 20;
        long start = System.nanoTime();
        IntStream.range(0, runs).parallel().forEach(i -> {
            sigAndMsg.get().writePosition(0);
            Ed25519.sign(sigAndMsg.get(), secretKey, secretKey);
            sigAndMsg2.get().writePosition(0);
            Ed25519.sign(sigAndMsg2.get(), privateKey, secretKey);
        });
        long time = System.nanoTime() - start;

        Bytes<?> bytes = sigAndMsg.get();
        bytes.writePosition(0);
        Bytes<?> bytes2 = sigAndMsg2.get();
        bytes2.writePosition(0);
        Ed25519.sign(bytes, secretKey, secretKey);
        Ed25519.sign(bytes2, privateKey, secretKey);
        long start2 = System.nanoTime();
        IntStream.range(0, runs).parallel().forEach(i -> {
            assertTrue(Ed25519.verify(bytes, publicKey));
            assertTrue(Ed25519.verify(bytes2, publicKey));
        });
        long time2 = System.nanoTime() - start2;
        System.out.println(
                "Throughput: " + "Sign: " + (long) ((2 * runs * 1e9) / time) + "/s, " + "Verify: " + (long) ((2 * runs * 1e9) / time2) + "/s");
        Jvm.pause(100);
    }
}