net.openhft.chronicle.core.Jvm Java Examples

private static void logToStandardOutMessageReceivedInERROR(@NotNull final Wire wire) {
    @NotNull final Bytes<?> bytes = wire.bytes();

    final long position = bytes.writePosition();
    final long limit = bytes.writeLimit();
    try {
        try {

            LOG.info("\nreceives IN ERROR:\n" +
                    "```yaml\n" +
                    Wires.fromSizePrefixedBlobs(wire) +
                    "```\n");
            YamlLogging.title = "";
            YamlLogging.writeMessage("");

        } catch (Exception e) {
            String x = Bytes.toString(bytes);
            Jvm.warn().on(TcpChannelHub.class, x, e);
        }
    } finally {
        bytes.writeLimit(limit);
        bytes.writePosition(position);
    }
}
 

boolean checkWritePositionHeaderNumber() {
    if (wire == null || wire.headerNumber() == Long.MIN_VALUE) return true;
    try {
        long pos = positionOfHeader;

        long seq1 = queue.rollCycle().toSequenceNumber(wire.headerNumber() + 1) - 1;
        long seq2 = store.sequenceForPosition(this, pos, true);

        if (seq1 != seq2) {
            String message = "~~~~~~~~~~~~~~ " +
                    "thread: " + Thread.currentThread().getName() +
                    " pos: " + pos +
                    " header: " + wire.headerNumber() +
                    " seq1: " + seq1 +
                    " seq2: " + seq2;
            AssertionError ae = new AssertionError(message);
            ae.printStackTrace();
            throw ae;
        }
    } catch (Exception e) {
        // TODO FIX
        Jvm.warn().on(getClass(), e);
        throw Jvm.rethrow(e);
    }
    return true;
}
 

public StartAndMonitor(ChronicleQueue queue, String name, int writePauseMs, int sleepBetweenMillis) {
            final SlowToSerialiseAndDeserialise object = new SlowToSerialiseAndDeserialise(writePauseMs);
            Thread thread = new Thread(() -> {
                try {
                    while (running.get()) {
                        long loopStart = System.nanoTime();
                        final ExcerptAppender appender = queue.acquireAppender();
//                        System.out.println("about to open");
                        try (final DocumentContext ctx = appender.writingDocument()) {
//                            System.out.println("about to write");
                            ctx.wire().getValueOut().marshallable(object);
//                            System.out.println("about to close");
                        }
//                        System.out.println("closed");
                        long timeTaken = System.nanoTime() - loopStart;
                        histo.sampleNanos(timeTaken);
                        Jvm.pause(sleepBetweenMillis);
                    }
                } catch (Throwable t) {
                    t.printStackTrace();
                }
            }, name);
            thread.start();
        }
 

private void logToStandardOutMessageSent(@NotNull final WireOut wire, @NotNull final ByteBuffer outBuffer) {
    if (!YamlLogging.showClientWrites())
        return;

    @NotNull Bytes<?> bytes = wire.bytes();

    try {

        if (bytes.readRemaining() > 0)
            LOG.info(((!YamlLogging.title.isEmpty()) ? "### " + YamlLogging
                    .title + "\n" : "") + "" +
                    YamlLogging.writeMessage() + (YamlLogging.writeMessage().isEmpty() ?
                    "" : "\n\n") +
                    "sends:\n\n" +
                    "```yaml\n" +
                    Wires.fromSizePrefixedBlobs(bytes) +
                    "```");
        YamlLogging.title = "";
        YamlLogging.writeMessage("");

    } catch (Exception e) {
        Jvm.warn().on(TcpChannelHub.class, Bytes.toString(bytes), e);
    }
}
 

StoreAppender(@NotNull final SingleChronicleQueue queue,
              @NotNull final WireStorePool storePool,
              final boolean checkInterrupts) {
    this.queue = queue;
    this.storePool = storePool;
    this.checkInterrupts = checkInterrupts;
    this.writeLock = queue.writeLock();
    this.context = new StoreAppenderContext();

    // always put references to "this" last.
    queue.addCloseListener(this);

    queue.cleanupStoreFilesWithNoData();
    int cycle = queue.cycle();
    int lastCycle = queue.lastCycle();
    if (lastCycle != cycle && lastCycle >= 0)
        // ensure that the EOF is written on the last cycle
        setCycle2(lastCycle, false);
    finalizer = Jvm.isResourceTracing() ? new Finalizer() : null;
}
 

@Override
public void refresh() {
    throwExceptionIfClosed();

    if (readOnly) {
        return;
    }
    while (true) {
        long currentMax = maxCycleValue.getVolatileValue();
        final File[] queueFiles = queuePath.toFile().
                listFiles((d, f) -> f.endsWith(SingleChronicleQueue.SUFFIX));
        int min = UNSET_MIN_CYCLE;
        int max = UNSET_MAX_CYCLE;
        if (queueFiles != null) {
            for (File queueFile : queueFiles) {
                min = Math.min(fileToCycleFunction.applyAsInt(queueFile), min);
                max = Math.max(fileToCycleFunction.applyAsInt(queueFile), max);
            }
        }
        minCycleValue.setOrderedValue(min);
        if (maxCycleValue.compareAndSwapValue(currentMax, max))
            break;
        Jvm.nanoPause();
    }
}
 

/**
 * Extra checks that get done for all test cases.
 * <p>
 * <p>Triggers test case Assert.failure if any thread assertions have Assert.failed,
 * by rethrowing, in the test harness thread, any exception recorded
 * earlier by threadRecordFailure.
 * <p>
 * <p>Triggers test case Assert.failure if interrupt status is set in the main thread.
 */
@After
public void tearDown() throws InterruptedException {
    Throwable t = threadFailure.getAndSet(null);
    if (t != null) {
        if (t instanceof Error)
            throw (Error) t;
        else if (t instanceof RuntimeException)
            throw (RuntimeException) t;
        else if (t instanceof Exception)
            throw Jvm.rethrow(t);
        else {
            AssertionFailedError afe =
                    new AssertionFailedError(t.toString());
            afe.initCause(t);
            throw afe;
        }
    }

    if (Thread.interrupted())
        throw new AssertionFailedError("interrupt status set in main thread");

    checkForkJoinPoolThreadLeaks();
    System.gc();
}
 

@Test
public void test() {
    File dir = getTmpDir();
    try (final ChronicleQueue queue = SingleChronicleQueueBuilder.builder(dir, wireType)
            .testBlockSize()
            .build()) {

        final ExcerptTailer tailer = queue.createTailer();
        assertFalse(tailer.readDocument(r -> r.read(TestKey.test).int32()));

        final ExcerptAppender appender = queue.acquireAppender();
        appender.writeDocument(w -> w.write(TestKey.test).int32(1));
        Jvm.pause(100);
        assertTrue(tailer.readDocument(r -> r.read(TestKey.test).int32()));
    }
}
 

public <R> R check(Call instance) {
    R r1 = null;
    R r2 = null;
    for (int i = 0; i < 50; i++) {
        r1 = (R) instance.method(map1);
        r2 = (R) instance.method(map2);

        if (r1 != null && r1.equals(r2))
            return r1;

        if (i > 30) {
            Jvm.pause(i);
        } else {
            Thread.yield();
        }
    }

    Assert.assertEquals(map1, map2);
    System.out.print(map1);
    System.out.print(map2);

    if (r1 != null)
        Assert.assertEquals(r1.toString(), r2.toString());

    return (R) r1;
}
 

public static void waitTillEqual(Map map1, Map map2, int timeOutMs) {
    int numberOfTimesTheSame = 0;
    long startTime = System.currentTimeMillis();
    for (int t = 0; t < timeOutMs + 100; t++) {
        // not map1.equals(map2), the reason is described above
        if (map1.equals(map2)) {
            numberOfTimesTheSame++;
            Jvm.pause(1);
            if (numberOfTimesTheSame == 10) {
                System.out.println("same");
                break;
            }
        }
        Jvm.pause(1);
        if (System.currentTimeMillis() - startTime > timeOutMs)
            break;
    }
}
 

private void waitForTheHeaderToBeBuilt(@NotNull Bytes bytes) throws IOException {
    for (int i = 0; i < 1000; i++) {
        long magic = bytes.readVolatileLong(MAGIC_OFFSET);
        if (magic == BUILDING) {
            try {
                Jvm.pause(10);
            } catch (InterruptedException e) {
                throw new IOException("Interrupted waiting for the header to be built");
            }
        } else if (magic == QUEUE_CREATED) {
            return;

        } else {
            throw new AssertionError("Invalid magic number " + Long.toHexString(magic) + " in file " + name());
        }
    }
    throw new AssertionError("Timeout waiting to build the file " + name());
}
 

/**
 * This method does not update the index, as indexes are not used for meta data
 *
 * @param buffer
 * @return the addressForRead of the appended data
 */
private long appendMetaDataReturnAddress(@NotNull Bytes buffer) {
    long length = checkRemainingForAppend(buffer);

    LongValue writeByte = header.writeByte();
    long lastByte = writeByte.getVolatileValue();

    for (; ; ) {
        if (bytes.compareAndSwapInt(lastByte, 0, NOT_COMPLETE | (int) length)) {
            long lastByte2 = lastByte + 4 + buffer.remaining();
            bytes.write(lastByte + 4, buffer);
            writeByte.setOrderedValue(lastByte2);
            bytes.writeOrderedInt(lastByte, (int) (META_DATA | length));
            return lastByte;
        }
        int length2 = length30(bytes.readVolatileInt());
        bytes.skip(length2);
        try {
            Jvm.checkInterrupted();
        } catch (InterruptedException e) {
            throw new InterruptedRuntimeException(e);
        }
    }
}
 

@SuppressWarnings("resource")
@Override
public Throwable call() {
    ChronicleQueue queue0 = null;
    try (ChronicleQueue queue = queueBuilder(path).build()) {
        queue0 = queue;
        ExcerptAppender appender = queue.acquireAppender();
        System.out.println("Starting pretoucher");
        while (!Thread.currentThread().isInterrupted() && !queue.isClosed()) {
            Jvm.pause(50);
            appender.pretouch();
        }
    } catch (Throwable e) {
        if (queue0 != null && queue0.isClosed())
            return null;
        exception = e;
        return e;
    }
    return null;
}
 

@Override
public boolean writeEOF(@NotNull Wire wire, long timeoutMS) {
    throwExceptionIfClosed();

    String fileName = mappedFile.file().getAbsolutePath();

    // just in case we are about to release this
    if (wire.bytes().tryReserve(this)) {
        try {
            return writeEOFAndShrink(wire, timeoutMS);

        } finally {
            wire.bytes().release(this);
        }
    }

    try (MappedBytes bytes = MappedBytes.mappedBytes(mappedFile.file(), mappedFile.chunkSize())) {
        Wire wire0 = WireType.valueOf(wire).apply(bytes);
        return writeEOFAndShrink(wire0, timeoutMS);

    } catch (Exception e) {
        Jvm.warn().on(getClass(), "unable to write the EOF file=" + fileName, e);
        return false;
    }
}
 

@Override
public long lastIndexAppended() {
    throwExceptionIfClosed();

    if (lastIndex != Long.MIN_VALUE)
        return lastIndex;

    if (lastPosition == Long.MIN_VALUE || wire == null) {
        throw new IllegalStateException("nothing has been appended, so there is no last index");
    }

    try {
        long sequenceNumber = store.sequenceForPosition(this, lastPosition, true);
        long index = queue.rollCycle().toIndex(lastCycle, sequenceNumber);
        lastIndex(index);
        return index;
    } catch (Exception e) {
        throw Jvm.rethrow(e);
    }
}
 

private boolean headerNumberCheck(@NotNull final AbstractWire wire) {

        wire.headNumberCheck((actual, position) -> {
            try {
                final long expecting = store.sequenceForPosition(this, position, false);
                if (actual == expecting)
                    return true;
                Jvm.warn().on(getClass(), new AssertionError("header number check failed " +
                        "expecting=" + expecting +
                        "  !=  actual=" + actual));

                return false;
            } catch (Exception e) {
                Jvm.warn().on(getClass(), "", e);
                return false;
            }
        });

        return true;
    }
 

private void log0(@NotNull ByteBuffer bytes, int start, int end) {
    @NotNull final StringBuilder sb = new StringBuilder(desc);
    sb.append(" len: ").append(end - start).append(" - ");
    if (end - start > 128) {
        for (int i = start; i < start + 64; i++)
            appendByte(bytes, sb, i);
        sb.append(" ... ");
        for (int i = end - 64; i < end; i++)
            appendByte(bytes, sb, i);
    } else {
        for (int i = start; i < end; i++)
            appendByte(bytes, sb, i);
    }

    Jvm.debug().on(getClass(), sb.toString());
}
 

protected void writeBytesInternal(final long index, @NotNull final BytesStore bytes, boolean metadata) {
    final int cycle = queue.rollCycle().toCycle(index);

    if (wire == null)
        setCycle2(cycle, true);
    else if (this.cycle < cycle)
        rollCycleTo(cycle);

    boolean rollbackDontClose = index != wire.headerNumber() + 1;
    if (rollbackDontClose) {
        if (index > wire.headerNumber() + 1)
            throw new IllegalStateException("Unable to move to index " + Long.toHexString(index) + " beyond the end of the queue, current: " + Long.toHexString(wire.headerNumber()));
        Jvm.warn().on(getClass(), "Trying to overwrite index " + Long.toHexString(index) + " which is before the end of the queue", new StackTrace());
        return;
    }
    writeBytesInternal(bytes, metadata);
}
 

protected void initializeMetadata() {
    File metapath = metapath();
    validateRollCycle(metapath);
    SCQMeta metadata = new SCQMeta(new SCQRoll(rollCycle(), epoch(), rollTime, rollTimeZone), deltaCheckpointInterval(),
            sourceId());
    try {

        boolean readOnly = readOnly();
        metaStore = SingleTableBuilder.binary(metapath, metadata).readOnly(readOnly).build();
        // check if metadata was overridden
        SCQMeta newMeta = metaStore.metadata();
        if (sourceId() == 0)
            sourceId(newMeta.sourceId());

        String format = newMeta.roll().format();
        if (!format.equals(rollCycle().format())) {
            // roll cycle changed
            overrideRollCycleForFileName(format);
        }

        // if it was overridden - reset
        rollTime = newMeta.roll().rollTime();
        rollTimeZone = newMeta.roll().rollTimeZone();
        epoch = newMeta.roll().epoch();
    } catch (IORuntimeException ex) {
        // readonly=true and file doesn't exist
        if (OS.isWindows())
            throw ex; // we cant have a read-only table store on windows so we have no option but to throw the ex.
        Jvm.warn().on(getClass(), "Failback to readonly tablestore", ex);
        metaStore = new ReadonlyTableStore<>(metadata);
    }
}
 

public static void release(File file) throws IOException {
    canonicalRafs.computeIfPresent(file, (f, ref) -> {
        if (--ref.refCount == 0) {
            try {
                ref.raf.close();
            } catch (IOException e) {
                throw Jvm.rethrow(e);
            }
            return null;
        } else {
            return ref;
        }
    });
}
 

private static int getNativeFileDescriptor(FileDescriptor descriptor) {
    try {
        final Field field = descriptor.getClass().getDeclaredField("fd");
        Jvm.setAccessible(field);
        return (int) field.get(descriptor);
    } catch (final Exception e) {
        LOG.warn("unsupported FileDescriptor implementation: e={}", e.getLocalizedMessage());
        return -1;
    }
}
 

private void pauseUnderload() {
    // when mapping and unmapping sections really fast it appears the OS/CPU gets confused as to whether memory is valid.
    long now = System.currentTimeMillis();
    if (now - lastTimeMapped < 5)
        Jvm.pause(2);
    lastTimeMapped = now;
}
 

@PackageLocal
void incrementIndex() {
    final RollCycle rollCycle = queue.rollCycle();
    final long index = this.index();
    long seq = rollCycle.toSequenceNumber(index);
    final int cycle = rollCycle.toCycle(index);

    seq += direction.add();
    switch (direction) {
        case NONE:
            break;
        case FORWARD:
            // if it runs out of seq number it will flow over to tomorrows cycle file
            if (rollCycle.toSequenceNumber(seq) < seq) {
                cycle(cycle + 1);
                Jvm.warn().on(getClass(),
                        "we have run out of sequence numbers, so will start to write to " +
                                "the next .cq4 file, the new cycle=" + cycle);
                seq = 0;
            }
            break;
        case BACKWARD:
            if (seq < 0) {
                windBackCycle(cycle);
                return;
            } else if (seq > 0 && striding) {
                seq -= seq % rollCycle.defaultIndexSpacing();
            }
            break;
    }
    index0(rollCycle.toIndex(cycle, seq));

}
 

public void install() {
    Set<Integer> hostIds = hostDetails.values().stream().map(HostDetails::hostId).collect(Collectors.toSet());

    int local = context.localIdentifier();
    if (!hostIds.contains(local)) {
        if (Jvm.isDebugEnabled(getClass()))
            Jvm.debug().on(getClass(), "cluster='" + context.clusterName() + "' ignored as localIdentifier=" + context.localIdentifier() + " is in this cluster");
        return;
    }

    if (context != null)
        hostDetails.values().forEach(context);
}
 

public static RandomAccessFile acquire(File file) throws FileNotFoundException {
    return canonicalRafs.compute(file, (f, ref) -> {
        if (ref == null) {
            try {
                return new RafReference(new CleaningRandomAccessFile(f, "rw"));
            } catch (FileNotFoundException e) {
                throw Jvm.rethrow(e);
            }
        } else {
            ref.refCount++;
            return ref;
        }
    }).raf;
}
 

/**
 * Returns a non-decreasing number, assumed to be used as a "timestamp".
 * <p>
 * <p>Approximate system time interval between two calls of this method is retrievable via
 * {@link #systemTimeIntervalBetween(long, long, TimeUnit)}, applied to the returned values
 * from those {@code currentTime()} calls.
 * <p>
 * <p>Safe and scalable for concurrent use from multiple threads.
 *
 * @return the current timestamp
 */
public static long currentTime() {
    long now = MILLISECONDS.toNanos(millisecondSupplier.getAsLong());
    while (true) {
        long lastTime = lastTimeHolder.get();
        if (now <= lastTime)
            return lastTime;
        if (lastTimeHolder.compareAndSet(lastTime, now))
            return now;
        Jvm.nanoPause();
    }
}
 

/**
 * When Chronicle Maps are created using {@link #createPersistedTo(File)} or
 * {@link #recoverPersistedTo(File, boolean)} or {@link
 * #createOrRecoverPersistedTo(File, boolean)} methods, file lock on the Chronicle Map's file is
 * acquired, that shouldn't be done from concurrent threads within the same JVM process. So
 * creation of Chronicle Maps persisted to the same File should be synchronized across JVM's
 * threads. Simple way would be to synchronize on some static (lock) object, but would serialize
 * all Chronicle Maps creations (persisted to any files), ConcurrentHashMap#compute() gives more
 * scalability. ConcurrentHashMap is used effectively for lock striping only, because the
 * entries are not even landing the map, because compute() always returns null.
 */
private static void fileLockedIO(
        File file, FileChannel fileChannel, FileIOAction fileIOAction) {
    fileLockingControl.compute(file, (k, v) -> {
        try {
            try (FileLock ignored = fileChannel.lock()) {
                fileIOAction.fileIOAction();
            }
            return null;
        } catch (IOException e) {
            throw Jvm.rethrow(e);
        }
    });
}
 

@Override
public <T extends Metadata> void overrideFrom(T metadata) {
    if (!(metadata instanceof SCQMeta))
        throw new IllegalStateException("Expected SCQMeta, got " + metadata.getClass());

    SCQMeta other = (SCQMeta) metadata;

    SCQRoll roll = other.roll;
    if (roll.epoch() != this.roll.epoch()) {
        Jvm.warn().on(getClass(), "Overriding roll epoch from existing metadata, was " + this.roll.epoch() + ", overriding to " + roll.epoch());
        this.roll.epoch(roll.epoch());
    }

    if (roll.length() != this.roll.length()) {
        Jvm.warn().on(getClass(), "Overriding roll length from existing metadata, was " + this.roll.length() + ", overriding to " + roll.length());
        this.roll.length(roll.length());
        this.roll.format(roll.format());
    }

    if (roll.rollTime() != null && !Objects.equals(roll.rollTime(), this.roll.rollTime())) {
        Jvm.warn().on(getClass(), "Overriding roll time from existing metadata, was " + this.roll.rollTime() + ", overriding to " + roll.rollTime());
        this.roll.rollTime(roll.rollTime());
    }

    if (roll.rollTimeZone() != null && !Objects.equals(roll.rollTimeZone(), this.roll.rollTimeZone())) {
        Jvm.warn().on(getClass(), "Overriding roll time zone from existing metadata, was " + this.roll.rollTimeZone() + ", overriding to " + roll.rollTimeZone());
        this.roll.rollTimeZone(roll.rollTimeZone());
    }

    if (!(other.sourceId == 0 || sourceId == 0 || other.sourceId == sourceId)) {
        Jvm.warn().on(getClass(), "inconsistency with of source ids, existing sourceId=" + other.sourceId + ", requested sourceId=" + sourceId);
    }
}
 

@Test(timeout = 5000)
public void readOnlyQueueTailerShouldObserveChangesAfterInitiallyObservedReadLimit() throws IOException, InterruptedException, TimeoutException, ExecutionException {
    DirectoryUtils.deleteDir(dataDir.toFile());
    dataDir.toFile().mkdirs();
    try (final ChronicleQueue queue = SingleChronicleQueueBuilder.binary(dataDir).testBlockSize().build()) {

        final StringEvents events = queue.acquireAppender().methodWriterBuilder(StringEvents.class).build();
        events.say("hello");

        final long readerCapacity = getCurrentQueueFileLength(dataDir);

        final RecordCounter recordCounter = new RecordCounter();
        final ChronicleReader chronicleReader = basicReader().withMessageSink(recordCounter);

        final ExecutorService executorService = Executors.newSingleThreadExecutor(
                new NamedThreadFactory("executor"));
        Future<?> submit = executorService.submit(chronicleReader::execute);

        final long expectedReadingDocumentCount = (readerCapacity / ONE_KILOBYTE.length) + 1;
        int i;
        for (i = 0; i < expectedReadingDocumentCount; i++) {
            events.say(new String(ONE_KILOBYTE));
        }

        recordCounter.latch.countDown();
        executorService.shutdown();
        executorService.awaitTermination(Jvm.isDebug() ? 50 : 5, TimeUnit.SECONDS);
        submit.get(1, TimeUnit.SECONDS);

        // #460 read only not supported on windows.
        if (!OS.isWindows())
            assertEquals(expectedReadingDocumentCount, recordCounter.recordCount.get() - 1);
    }
}
 

/**
 * called periodically to check that the heartbeat has been received
 *
 * @return {@code true} if we have received a heartbeat recently
 */
private boolean hasReceivedHeartbeat() {
    long currentTimeMillis = System.currentTimeMillis();
    boolean result = lastTimeMessageReceived + heartbeatTimeoutMs >= currentTimeMillis;

    if (!result)
        Jvm.warn().on(getClass(), Integer.toHexString(hashCode()) + " missed heartbeat, lastTimeMessageReceived=" + lastTimeMessageReceived
                + ", currentTimeMillis=" + currentTimeMillis);
    return result;
}