Java Code Examples for org.apache.distributedlog.LogRecordWithDLSN#getDlsn()

The following examples show how to use org.apache.distributedlog.LogRecordWithDLSN#getDlsn() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ReaderWorker.java    From distributedlog with Apache License 2.0 6 votes vote down vote up 
public void processRecord(final LogRecordWithDLSN record) {
    Message msg;
    try {
        msg = Utils.parseMessage(record.getPayload());
    } catch (TException e) {
        invalidRecordsCounter.inc();
        LOG.warn("Failed to parse record {} for stream {} : size = {} , ",
                 new Object[] { record, streamIdx, record.getPayload().length, e });
        return;
    }
    long curTimeMillis = System.currentTimeMillis();
    long e2eLatency = curTimeMillis - msg.getPublishTime();
    long deliveryLatency = curTimeMillis - record.getTransactionId();
    if (e2eLatency >= 0) {
        e2eStat.registerSuccessfulEvent(e2eLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeE2EStat.registerSuccessfulEvent(-e2eLatency, TimeUnit.MILLISECONDS);
    }
    if (deliveryLatency >= 0) {
        deliveryStat.registerSuccessfulEvent(deliveryLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeDeliveryStat.registerSuccessfulEvent(-deliveryLatency, TimeUnit.MILLISECONDS);
    }

    prevDLSN = record.getDlsn();
}
Example 2
Source File: LogSegmentMetadataStoreUpdater.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
@Override
public CompletableFuture<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment,
                                                              LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    checkState(!segment.isInProgress(),
            "Updating last dlsn for an inprogress log segment isn't supported.");
    checkArgument(segment.isDLSNinThisSegment(dlsn),
            "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator()
            .setLastDLSN(dlsn)
            .setLastTxId(record.getTransactionId())
            .setRecordCount(record)
            .build();
    return updateSegmentMetadata(newSegment);
}
Example 3
Source File: TailReader.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // get the last record
    LogRecordWithDLSN lastRecord;
    DLSN dlsn;
    try {
        lastRecord = dlm.getLastLogRecord();
        dlsn = lastRecord.getDlsn();
        readLoop(dlm, dlsn);
    } catch (LogNotFoundException lnfe) {
        System.err.println("Log stream " + streamName + " is not found. Please create it first.");
        return;
    } catch (LogEmptyException lee) {
        System.err.println("Log stream " + streamName + " is empty.");
        dlsn = DLSN.InitialDLSN;
        readLoop(dlm, dlsn);
    } finally {
        dlm.close();
        namespace.close();
    }
}
Example 4
Source File: StreamTransformer.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private static void transform(final AsyncLogWriter writer,
                              LogRecordWithDLSN record,
                              Transformer<byte[], byte[]> replicationTransformer,
                              final CountDownLatch keepAliveLatch)
        throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord =
            new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data))
            .whenComplete(new FutureEventListener<DLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}
Example 5
Source File: TestBKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testReadEntriesFromCompleteLogSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();
    boolean done = false;
    long txId = 1L;
    long entryId = 0L;
    while (!done) {
        Entry.Reader entryReader;
        try {
            entryReader = Utils.ioResult(reader.readNext(1)).get(0);
        } catch (EndOfLogSegmentException eol) {
            done = true;
            continue;
        }
        LogRecordWithDLSN record = entryReader.nextRecord();
        while (null != record) {
            if (!record.isControl()) {
                DLMTestUtil.verifyLogRecord(record);
                assertEquals(txId, record.getTransactionId());
                ++txId;
            }
            DLSN dlsn = record.getDlsn();
            assertEquals(1L, dlsn.getLogSegmentSequenceNo());
            assertEquals(entryId, dlsn.getEntryId());
            record = entryReader.nextRecord();
        }
        ++entryId;
    }
    assertEquals(21, txId);
    assertFalse(reader.hasCaughtUpOnInprogress());
    Utils.close(reader);
}
Example 6
Source File: TestBKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(2);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(10, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 10) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(10, reader.readAheadEntries.size());
    assertEquals(11, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
Example 7
Source File: TestBKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testMaxPrefetchEntriesLargeBatch() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(5);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(5, reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    // wait for the read ahead entries to become 10 again
    while (reader.readAheadEntries.size() < 5) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    assertEquals(5, reader.readAheadEntries.size());
    assertEquals(6, reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}
Example 8
Source File: TestBKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 4 votes vote down vote up 
@Test(timeout = 60000)
public void testMaxPrefetchEntriesSmallSegment() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(20);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    generateCompletedLogSegments(dlm, confLocal, 1, 5);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    reader.start();

    // wait for the read ahead entries to become available
    while (reader.readAheadEntries.size() < (reader.getLastAddConfirmed() + 1)) {
        TimeUnit.MILLISECONDS.sleep(10);
    }

    long txId = 1L;
    long entryId = 0L;

    assertEquals((reader.getLastAddConfirmed() + 1), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    // read first entry
    Entry.Reader entryReader = Utils.ioResult(reader.readNext(1)).get(0);
    LogRecordWithDLSN record = entryReader.nextRecord();
    while (null != record) {
        if (!record.isControl()) {
            DLMTestUtil.verifyLogRecord(record);
            assertEquals(txId, record.getTransactionId());
            ++txId;
        }
        DLSN dlsn = record.getDlsn();
        assertEquals(1L, dlsn.getLogSegmentSequenceNo());
        assertEquals(entryId, dlsn.getEntryId());
        record = entryReader.nextRecord();
    }
    ++entryId;
    assertEquals(2L, txId);
    assertEquals(reader.getLastAddConfirmed(), reader.readAheadEntries.size());
    assertEquals((reader.getLastAddConfirmed() + 1), reader.getNextEntryId());
    assertFalse(reader.hasCaughtUpOnInprogress());

    Utils.close(reader);
}