org.apache.distributedlog.LogRecordWithDLSN Java Examples

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  LogReader reader = mock(LogReader.class);
  when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

  byte[] data = "test-read".getBytes(UTF_8);
  LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
  when(record.getPayLoadInputStream())
      .thenReturn(new ByteArrayInputStream(data));
  when(reader.readNext(anyBoolean()))
      .thenReturn(record)
      .thenThrow(new EndOfStreamException("eos"));

  DLInputStream in = new DLInputStream(dlm);
  int numReads = 0;
  int readByte;
  while ((readByte = in.read()) != -1) {
    assertEquals(data[numReads], readByte);
    ++numReads;
  }
  assertEquals(data.length, numReads);
}
 

/**
 * Test Case: read records from the input stream.
 */
@Test
public void testRead() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    byte[] data = "test-read".getBytes(UTF_8);
    LogRecordWithDLSN record = mock(LogRecordWithDLSN.class);
    when(record.getPayLoadInputStream())
        .thenReturn(new ByteArrayInputStream(data));
    when(reader.readNext(anyBoolean()))
        .thenReturn(record)
        .thenThrow(new EndOfStreamException("eos"));

    DLInputStream in = new DLInputStream(dlm);
    int numReads = 0;
    int readByte;
    while ((readByte = in.read()) != -1) {
        assertEquals(data[numReads], readByte);
        ++numReads;
    }
    assertEquals(data.length, numReads);
}
 

public void processRecord(final LogRecordWithDLSN record) {
    Message msg;
    try {
        msg = Utils.parseMessage(record.getPayload());
    } catch (TException e) {
        invalidRecordsCounter.inc();
        LOG.warn("Failed to parse record {} for stream {} : size = {} , ",
                 new Object[] { record, streamIdx, record.getPayload().length, e });
        return;
    }
    long curTimeMillis = System.currentTimeMillis();
    long e2eLatency = curTimeMillis - msg.getPublishTime();
    long deliveryLatency = curTimeMillis - record.getTransactionId();
    if (e2eLatency >= 0) {
        e2eStat.registerSuccessfulEvent(e2eLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeE2EStat.registerSuccessfulEvent(-e2eLatency, TimeUnit.MILLISECONDS);
    }
    if (deliveryLatency >= 0) {
        deliveryStat.registerSuccessfulEvent(deliveryLatency, TimeUnit.MILLISECONDS);
    } else {
        negativeDeliveryStat.registerSuccessfulEvent(-deliveryLatency, TimeUnit.MILLISECONDS);
    }

    prevDLSN = record.getDlsn();
}
 

private void dumpRecord(LogRecord record) {
    System.out.println("------------------------------------------------");
    if (record instanceof LogRecordWithDLSN) {
        System.out.println("Record (txn = " + record.getTransactionId() + ", bytes = "
                + record.getPayload().length + ", dlsn = "
                + ((LogRecordWithDLSN) record).getDlsn() + ", sequence id = "
                + ((LogRecordWithDLSN) record).getSequenceId() + ")");
    } else {
        System.out.println("Record (txn = " + record.getTransactionId() + ", bytes = "
                + record.getPayload().length + ")");
    }
    System.out.println("");

    if (skipPayload) {
        return;
    }

    if (printHex) {
        System.out.println(Hex.encodeHexString(record.getPayload()));
    } else {
        System.out.println(new String(record.getPayload(), UTF_8));
    }
}
 

@Test(timeout = 60000)
public void testToolTruncateStream() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setLogSegmentCacheEnabled(false);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", confLocal, defaultUri);
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 3, 1000);

    DLSN dlsn = new DLSN(2, 1, 0);
    TruncateStreamCommand cmd = new TruncateStreamCommand();
    cmd.setDlsn(dlsn);
    cmd.setUri(defaultUri);
    cmd.setStreamName("testToolTruncateStream");
    cmd.setForce(true);

    assertEquals(0, cmd.runCmd());

    LogReader reader = dlm.getInputStream(0);
    LogRecordWithDLSN record = reader.readNext(false);
    assertEquals(dlsn, record.getDlsn());

    reader.close();
    dlm.close();
}
 

@Override
public CompletableFuture<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment,
                                                              LogRecordWithDLSN record) {
    DLSN dlsn = record.getDlsn();
    checkState(!segment.isInProgress(),
            "Updating last dlsn for an inprogress log segment isn't supported.");
    checkArgument(segment.isDLSNinThisSegment(dlsn),
            "DLSN " + dlsn + " doesn't belong to segment " + segment);
    final LogSegmentMetadata newSegment = segment.mutator()
            .setLastDLSN(dlsn)
            .setLastTxId(record.getTransactionId())
            .setRecordCount(record)
            .build();
    return updateSegmentMetadata(newSegment);
}
 

protected void repairLogSegment(BookKeeperAdmin bkAdmin,
                                MetadataUpdater metadataUpdater,
                                LogSegmentMetadata segment) throws Exception {
    if (segment.isInProgress()) {
        System.out.println("Skip inprogress log segment " + segment);
        return;
    }
    LedgerHandle lh = bkAdmin.openLedger(segment.getLogSegmentId());
    long lac = lh.getLastAddConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(lac, lac);
    if (!entries.hasMoreElements()) {
        throw new IOException("Entry " + lac + " isn't found for " + segment);
    }
    LedgerEntry lastEntry = entries.nextElement();
    Entry.Reader reader = Entry.newBuilder()
            .setLogSegmentInfo(segment.getLogSegmentSequenceNumber(), segment.getStartSequenceId())
            .setEntryId(lastEntry.getEntryId())
            .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(segment.getVersion()))
            .setEntry(lastEntry.getEntryBuffer())
            .buildReader();
    lastEntry.getEntryBuffer().release();
    LogRecordWithDLSN record = reader.nextRecord();
    LogRecordWithDLSN lastRecord = null;
    while (null != record) {
        lastRecord = record;
        record = reader.nextRecord();
    }
    if (null == lastRecord) {
        throw new IOException("No record found in entry " + lac + " for " + segment);
    }
    System.out.println("Updating last record for " + segment + " to " + lastRecord);
    if (!IOUtils.confirmPrompt("Do you want to make this change (Y/N): ")) {
        return;
    }
    metadataUpdater.updateLastRecord(segment, lastRecord);
}
 

int countFromStartToEnd(DistributedLogManager dlm) throws Exception {
    int count = 0;
    try {
        LogReader reader = dlm.getInputStream(startDLSN);
        try {
            LogRecordWithDLSN record = reader.readNext(false);
            LogRecordWithDLSN preRecord = record;
            System.out.println("first record : " + record);
            while (null != record) {
                if (record.getDlsn().compareTo(endDLSN) > 0) {
                    break;
                }
                ++count;
                if (count % 1000 == 0) {
                    logger.info("read {} records from {}...", count, getStreamName());
                }
                preRecord = record;
                record = reader.readNext(false);
            }
            System.out.println("last record : " + preRecord);
        } finally {
            reader.close();
        }
    } finally {
        dlm.close();
    }
    return count;
}
 

private void printEntry(Entry.Reader reader) throws Exception {
    LogRecordWithDLSN record = reader.nextRecord();
    while (null != record) {
        System.out.println("\t" + record);
        if (!skipPayload) {
            if (printHex) {
                System.out.println(Hex.encodeHexString(record.getPayload()));
            } else {
                System.out.println(new String(record.getPayload(), UTF_8));
            }
        }
        System.out.println("");
        record = reader.nextRecord();
    }
}
 

private static RecordStream nextRecordStream(LogReader reader) throws IOException {
    LogRecordWithDLSN record = reader.readNext(false);
    if (null != record) {
        return new RecordStream(record);
    }
    return null;
}
 

public static void main(String[] args) throws Exception {
    if (2 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String streamName = args[1];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + streamName);
    DistributedLogManager dlm = namespace.openLog(streamName);

    // get the last record
    LogRecordWithDLSN lastRecord;
    DLSN dlsn;
    try {
        lastRecord = dlm.getLastLogRecord();
        dlsn = lastRecord.getDlsn();
        readLoop(dlm, dlsn);
    } catch (LogNotFoundException lnfe) {
        System.err.println("Log stream " + streamName + " is not found. Please create it first.");
        return;
    } catch (LogEmptyException lee) {
        System.err.println("Log stream " + streamName + " is empty.");
        dlsn = DLSN.InitialDLSN;
        readLoop(dlm, dlsn);
    } finally {
        dlm.close();
        namespace.close();
    }
}
 

private static void readLoop(final DistributedLogManager dlm,
                             final DLSN dlsn) throws Exception {

    final CountDownLatch keepAliveLatch = new CountDownLatch(1);

    System.out.println("Wait for records starting from " + dlsn);
    final AsyncLogReader reader = FutureUtils.result(dlm.openAsyncLogReader(dlsn));
    final FutureEventListener<LogRecordWithDLSN> readListener = new FutureEventListener<LogRecordWithDLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on reading records from stream " + dlm.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(LogRecordWithDLSN record) {
            System.out.println("Received record " + record.getDlsn());
            System.out.println("\"\"\"");
            System.out.println(new String(record.getPayload(), UTF_8));
            System.out.println("\"\"\"");
            reader.readNext().whenComplete(this);
        }
    };
    reader.readNext().whenComplete(readListener);

    keepAliveLatch.await();
    FutureUtils.result(reader.asyncClose(), 5, TimeUnit.SECONDS);
}
 

private static void readLoop(final DistributedLogManager dlm,
                             final DLSN dlsn,
                             final AtomicReference<DLSN> lastDLSN)
        throws Exception {

    final CountDownLatch keepAliveLatch = new CountDownLatch(1);

    System.out.println("Wait for records starting from " + dlsn);
    final AsyncLogReader reader = FutureUtils.result(dlm.openAsyncLogReader(dlsn));
    final FutureEventListener<LogRecordWithDLSN> readListener = new FutureEventListener<LogRecordWithDLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on reading records from stream " + dlm.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(LogRecordWithDLSN record) {
            System.out.println("Received record " + record.getDlsn());
            System.out.println("\"\"\"");
            System.out.println(new String(record.getPayload(), UTF_8));
            System.out.println("\"\"\"");
            lastDLSN.set(record.getDlsn());
            reader.readNext().whenComplete(this);
        }
    };
    reader.readNext().whenComplete(readListener);

    keepAliveLatch.await();
    FutureUtils.result(reader.asyncClose(), 5, TimeUnit.SECONDS);
}
 

private static void transform(final AsyncLogWriter writer,
                              LogRecordWithDLSN record,
                              Transformer<byte[], byte[]> replicationTransformer,
                              final CountDownLatch keepAliveLatch)
        throws Exception {
    DLSN srcDLSN = record.getDlsn();
    byte[] payload = record.getPayload();
    byte[] transformedPayload = replicationTransformer.transform(payload);
    TransformedRecord transformedRecord =
            new TransformedRecord(ByteBuffer.wrap(transformedPayload));
    transformedRecord.setSrcDlsn(srcDLSN.serializeBytes());
    ByteArrayOutputStream baos = new ByteArrayOutputStream(4096);
    transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos)));
    byte[] data = baos.toByteArray();
    writer.write(new LogRecord(record.getSequenceId(), data))
            .whenComplete(new FutureEventListener<DLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on writing records to stream " + writer.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(DLSN dlsn) {
            System.out.println("Write transformed record " + dlsn);
        }
    });
}
 

@Override
public void onSuccess(final List<LogRecordWithDLSN> records) {
    for (final LogRecordWithDLSN record : records) {
        if (record.isRecordSet()) {
            try {
                processRecordSet(record);
            } catch (IOException e) {
                onFailure(e);
            }
        } else {
            processRecord(record);
        }
    }
    readLoop();
}
 

public void processRecordSet(final LogRecordWithDLSN record) throws IOException {
    LogRecordSet.Reader reader = LogRecordSet.of(record);
    LogRecordWithDLSN nextRecord = reader.nextRecord();
    while (null != nextRecord) {
        processRecord(nextRecord);
        nextRecord = reader.nextRecord();
    }
    reader.release();
}
 

private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
 

private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException {
  LogRecordWithDLSN record = reader.readNext(false);

  if (null != record) {
    return new LogRecordWithInputStream(record);
  } else {
    record = reader.readNext(false);
    if (null != record) {
      return new LogRecordWithInputStream(record);
    } else {
      return null;
    }
  }
}
 

public static Operation getOperationFromLogRecord(LogRecordWithDLSN logRecord) throws IOException {
    ByteBuffer bf = ByteBuffer.wrap(logRecord.getPayload());
    ByteBufferStreamInput in = new ByteBufferStreamInput(bf);
    Translog.Operation.Type type = Translog.Operation.Type.fromId(in.readByte());
    Translog.Operation operation = newOperationFromType(type);
    operation.readFrom(in);
    in.close();
    return operation;
}
 

private static CompletableFuture<LogSegmentCandidate> checkLogSegment(
        final Namespace namespace,
        final String streamName,
        final LogSegmentMetadata metadata,
        final OrderedScheduler scheduler) {
    if (metadata.isInProgress()) {
        return FutureUtils.value(null);
    }

    final LogSegmentEntryStore entryStore = namespace.getNamespaceDriver()
            .getLogSegmentEntryStore(NamespaceDriver.Role.READER);
    return ReadUtils.asyncReadLastRecord(
            streamName,
            metadata,
            true,
            false,
            true,
            4,
            16,
            new AtomicInteger(0),
            scheduler,
            entryStore
    ).thenApply(new Function<LogRecordWithDLSN, LogSegmentCandidate>() {
        @Override
        public LogSegmentCandidate apply(LogRecordWithDLSN record) {
            if (null != record
                    && (record.getDlsn().compareTo(metadata.getLastDLSN()) > 0
                        || record.getTransactionId() > metadata.getLastTxId()
                        || !metadata.isRecordPositionWithinSegmentScope(record))) {
                return new LogSegmentCandidate(metadata, record);
            } else {
                return null;
            }
        }
    });
}
 

@Override
public void process(LogRecordWithDLSN record) {
    if (null == this.firstRecord
            && (includeControl || !record.isControl())) {
        this.firstRecord = record;
    }
}
 

@Override
public void process(LogRecordWithDLSN record) {
    if (found) {
        return;
    }
    this.result = record;
    if (record.getTransactionId() >= txId) {
        found = true;
    }
}
 

@Override
public void process(LogRecordWithDLSN record) {
    lastRecord = record;
}
 

LogRecordWithDLSN getLogRecord() {
  return logRecord;
}
 

LogRecordWithInputStream(LogRecordWithDLSN logRecord) {
  this.logRecord = logRecord;
  this.payloadStream = logRecord.getPayLoadInputStream();
}
 

LogSegmentCandidate(LogSegmentMetadata metadata, LogRecordWithDLSN lastRecord) {
    this.metadata = metadata;
    this.lastRecord = lastRecord;
}
 

LogRecordWithDLSN getLogRecord() {
  return logRecord;
}
 

LogRecordWithInputStream(LogRecordWithDLSN logRecord) {
  this.logRecord = logRecord;
  this.payloadStream = logRecord.getPayLoadInputStream();
}
 

private static void readLoop(final DistributedLogManager dlm,
                             final DLSN fromDLSN,
                             final AsyncLogWriter targetWriter,
                             final Transformer<byte[], byte[]> replicationTransformer)
        throws Exception {

    final CountDownLatch keepAliveLatch = new CountDownLatch(1);

    System.out.println("Wait for records starting from " + fromDLSN);
    final AsyncLogReader reader = FutureUtils.result(dlm.openAsyncLogReader(fromDLSN));
    final FutureEventListener<LogRecordWithDLSN> readListener = new FutureEventListener<LogRecordWithDLSN>() {
        @Override
        public void onFailure(Throwable cause) {
            System.err.println("Encountered error on reading records from stream " + dlm.getStreamName());
            cause.printStackTrace(System.err);
            keepAliveLatch.countDown();
        }

        @Override
        public void onSuccess(LogRecordWithDLSN record) {
            if (record.getDlsn().compareTo(fromDLSN) <= 0) {
                reader.readNext().whenComplete(this);
                return;
            }
            System.out.println("Received record " + record.getDlsn());
            System.out.println("\"\"\"");
            System.out.println(new String(record.getPayload(), UTF_8));
            System.out.println("\"\"\"");
            try {
                transform(targetWriter, record, replicationTransformer, keepAliveLatch);
            } catch (Exception e) {
                System.err.println("Encountered error on transforming record " + record.getDlsn()
                        + " from stream " + dlm.getStreamName());
                e.printStackTrace(System.err);
                keepAliveLatch.countDown();
            }
            reader.readNext().whenComplete(this);
        }
    };
    reader.readNext().whenComplete(readListener);

    keepAliveLatch.await();
    FutureUtils.result(reader.asyncClose(), 5, TimeUnit.SECONDS);
}
 

public static void main(String[] args) throws Exception {
    if (3 != args.length) {
        System.out.println(HELP);
        return;
    }

    String dlUriStr = args[0];
    final String srcStreamName = args[1];
    final String targetStreamName = args[2];

    URI uri = URI.create(dlUriStr);
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setOutputBufferSize(16*1024); // 16KB
    conf.setPeriodicFlushFrequencyMilliSeconds(5); // 5ms
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf)
            .uri(uri)
            .build();

    // open the dlm
    System.out.println("Opening log stream " + srcStreamName);
    DistributedLogManager srcDlm = namespace.openLog(srcStreamName);
    System.out.println("Opening log stream " + targetStreamName);
    DistributedLogManager targetDlm = namespace.openLog(targetStreamName);

    Transformer<byte[], byte[]> replicationTransformer =
            new IdenticalTransformer<byte[]>();

    LogRecordWithDLSN lastTargetRecord;
    DLSN srcDlsn;
    try {
        lastTargetRecord = targetDlm.getLastLogRecord();
        TransformedRecord lastTransformedRecord = new TransformedRecord();
        try {
            lastTransformedRecord.read(protocolFactory.getProtocol(
                    new TIOStreamTransport(new ByteArrayInputStream(lastTargetRecord.getPayload()))));
            srcDlsn = DLSN.deserializeBytes(lastTransformedRecord.getSrcDlsn());
            System.out.println("Last transformed record is " + srcDlsn);
        } catch (TException e) {
            System.err.println("Error on reading last transformed record");
            e.printStackTrace(System.err);
            srcDlsn = DLSN.InitialDLSN;
        }
    } catch (LogNotFoundException lnfe) {
        srcDlsn = DLSN.InitialDLSN;
    } catch (LogEmptyException lee) {
        srcDlsn = DLSN.InitialDLSN;
    }

    AsyncLogWriter targetWriter = FutureUtils.result(targetDlm.openAsyncLogWriter());
    try {
        readLoop(srcDlm, srcDlsn, targetWriter, replicationTransformer);
    } finally {
        FutureUtils.result(targetWriter.asyncClose(), 5, TimeUnit.SECONDS);
        targetDlm.close();
        srcDlm.close();
        namespace.close();
    }

}