org.apache.distributedlog.api.DistributedLogManager Java Examples

@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 

@Test(timeout = 60000)
public void testReaderLockSharedDlmDoesNotConflict() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();

    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);

    // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the
    // system currently works.
    Utils.ioResult(futureReader1);
    Utils.ioResult(futureReader2);

    dlm0.close();
    dlm1.close();
}
 

@Test(timeout = 60000)
public void testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(1 * 1000);
    conf.setImmediateFlushEnabled(false);
    conf.setOutputBufferSize(1024 * 1024);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();

    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    assertEquals(0, writer.position());

    Utils.ioResult(writer.write(byteStream));
    Thread.sleep(100); // let WriteCompleteListener have time to run
    assertEquals(33, writer.position());

    writer.close();
    dlmwriter.close();
}
 

private List<Long> collectLedgersFromStream(Namespace namespace,
                                            String stream,
                                            Set<Long> ledgers)
        throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLogSegmentId());
            }
            sLedgers.add(segment.getLogSegmentId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}
 

@Override
protected int runCmd() throws Exception {
    DistributedLogManager dlm = getNamespace().openLog(getStreamName());
    try {
        long count = 0;
        if (null == endDLSN) {
            count = countToLastRecord(dlm);
        } else {
            count = countFromStartToEnd(dlm);
        }
        System.out.println("total is " + count + " records.");
        return 0;
    } finally {
        dlm.close();
    }
}
 

@Override
public FSDataInputStream open(Path path, int bufferSize)
        throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path));
        LogReader reader;
        try {
            reader = dlm.openLogReader(DLSN.InitialDLSN);
        } catch (LogNotFoundException lnfe) {
            throw new FileNotFoundException(path.toString());
        } catch (LogEmptyException lee) {
            throw new FileNotFoundException(path.toString());
        }
        return new FSDataInputStream(
            new BufferedFSInputStream(
                new DLInputStream(dlm, reader, 0L),
                bufferSize));
    } catch (LogNotFoundException e) {
        throw new FileNotFoundException(path.toString());
    }
}
 

void generateCompletedLogSegments(DistributedLogManager dlm,
                                  long numCompletedSegments,
                                  long segmentSize,
                                  long startTxId) throws Exception {

    long txid = startTxId;
    for (long i = 0; i < numCompletedSegments; i++) {
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        for (long j = 1; j <= segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(txid);
            ctrlRecord.setControl();
            Utils.ioResult(writer.write(ctrlRecord));
        }
        Utils.close(writer);
    }
}
 

private void truncateStreams(Namespace namespace, List<String> streams,
                             int tid, int numStreamsPerThreads) throws IOException {
    int startIdx = tid * numStreamsPerThreads;
    int endIdx = Math.min(streams.size(), (tid + 1) * numStreamsPerThreads);
    for (int i = startIdx; i < endIdx; i++) {
        String s = streams.get(i);
        DistributedLogManager dlm = namespace.openLog(s);
        try {
            if (deleteStream) {
                dlm.delete();
            } else {
                dlm.purgeLogsOlderThan(Long.MAX_VALUE);
            }
        } finally {
            dlm.close();
        }
    }
}
 

@Test
public void testUploadPackageExisting() throws Exception {
  uploader = Mockito.spy(uploader);

  Namespace ns = mock(Namespace.class);
  when(nsBuilder.build()).thenReturn(ns);
  when(ns.logExists(anyString())).thenReturn(true);
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  when(ns.openLog(anyString())).thenReturn(dlm);
  AppendOnlyStreamWriter asw = mock(AppendOnlyStreamWriter.class);
  when(dlm.getAppendOnlyStreamWriter()).thenReturn(asw);

  Mockito.doReturn(true).when(uploader).isLocalFileExists(Mockito.anyString());

  uploader.initialize(config);
  uploader.uploadPackage();

  verify(ns, times(1)).deleteLog(eq(uploader.getPackageName()));
  verify(copier, times(1))
      .copyFileToStream(eq(uploader.getTopologyPackageLocation()), any(OutputStream.class));
  verify(asw, times(1)).close();
  verify(dlm, times(1)).close();
}
 

@Test(timeout = 60000)
public void testTwoWritersOnLockDisabled() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setWriteLockEnabled(false);
    String name = "distrlog-two-writers-lock-disabled";
    DistributedLogManager manager = createNewDLM(confLocal, name);
    AsyncLogWriter writer1 = Utils.ioResult(manager.openAsyncLogWriter());
    Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(1L)));
    AsyncLogWriter writer2 = Utils.ioResult(manager.openAsyncLogWriter());
    Utils.ioResult(writer2.write(DLMTestUtil.getLogRecordInstance(2L)));

    // write a record to writer 1 again
    try {
        Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(3L)));
        fail("Should fail writing record to writer 1 again as writer 2 took over the ownership");
    } catch (BKTransmitException bkte) {
        assertEquals(BKException.Code.LedgerFencedException, bkte.getBKResultCode());
    }
}
 

@Test
public void testStore() throws Exception {
  PowerMockito.mockStatic(CheckpointManager.InstanceStateCheckpoint.class);
  CheckpointManager.InstanceStateCheckpoint mockCheckpointState =
      mock(CheckpointManager.InstanceStateCheckpoint.class);

  final CheckpointInfo info = new CheckpointInfo(
      StatefulStorageTestContext.CHECKPOINT_ID, instance);
  Checkpoint checkpoint = new Checkpoint(mockCheckpointState);

  DistributedLogManager mockDLM = mock(DistributedLogManager.class);
  when(mockNamespace.openLog(anyString())).thenReturn(mockDLM);
  AppendOnlyStreamWriter mockWriter = mock(AppendOnlyStreamWriter.class);
  when(mockDLM.getAppendOnlyStreamWriter()).thenReturn(mockWriter);

  dlogStorage.storeCheckpoint(info, checkpoint);

  verify(mockWriter).markEndOfStream();
  verify(mockWriter).close();
}
 

public TestReader(String name,
                  DistributedLogManager dlm,
                  DLSN startDLSN,
                  boolean simulateErrors,
                  int delayMs,
                  CountDownLatch readyLatch,
                  CountDownLatch countLatch,
                  CountDownLatch completionLatch) {
    this.readerName = name;
    this.dlm = dlm;
    this.startDLSN = startDLSN;
    this.simulateErrors = simulateErrors;
    this.delayMs = delayMs;
    this.readyLatch = readyLatch;
    this.countLatch = countLatch;
    this.completionLatch = completionLatch;
    // States
    this.errorsFound = new AtomicBoolean(false);
    this.readCount = new AtomicInteger(0);
    this.positionReaderCount = new AtomicInteger(0);
    // Executors
    this.executorService = Executors.newSingleThreadScheduledExecutor();
}
 

/**
 * Write records into <i>numLogSegments</i> log segments.
 * Each log segment has <i>numRecordsPerLogSegment</i> records.
 *
 * @param dlm
 *          distributedlog manager
 * @param numLogSegments
 *          number of log segments
 * @param numRecordsPerLogSegment
 *          number records per log segment
 * @param startTxId
 *          start tx id
 * @return next tx id
 */
private static long writeRecords(DistributedLogManager dlm,
                                 int numLogSegments,
                                 int numRecordsPerLogSegment,
                                 long startTxId,
                                 boolean emptyRecord) throws IOException {
    long txid = startTxId;
    for (long i = 0; i < numLogSegments; i++) {
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= numRecordsPerLogSegment; j++) {
            if (emptyRecord) {
                writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++));
            } else {
                writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
            }
        }
        writer.closeAndComplete();
    }
    return txid;
}
 

/**
 * Test Case: A large write batch will span multiple packets.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testAsyncBulkWriteSpanningPackets() throws Exception {
    String name = "distrlog-testAsyncBulkWriteSpanningPackets";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());

    // First entry.
    int numTransmissions = 4;
    int recSize = 10 * 1024;
    int batchSize = (numTransmissions * MAX_LOGRECORDSET_SIZE + 1) / recSize;
    long ledgerIndex = 1;
    long entryIndex = 0;
    long slotIndex = 0;
    long txIndex = 1;
    DLSN dlsn = checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex);
    assertEquals(4, dlsn.getEntryId());
    assertEquals(1, dlsn.getLogSegmentSequenceNo());

    writer.closeAndComplete();
    dlm.close();
}
 

@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsDisabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(0);
    confLocal.setReadLACLongPollTimeout(9);
    confLocal.setReaderIdleWarnThresholdMillis(20);
    confLocal.setReaderIdleErrorThresholdMillis(40);

    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));

    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        Utils.ioResult(reader.readNext());
        fail("Should fail when stream is idle");
    } catch (IdleReaderException ire) {
        // expected
    }
    Utils.close(reader);
    writer.close();
    dlm.close();
}
 

@Test(timeout = 60000, expected = LogRecordTooLongException.class)
public void testMaxLogRecSize() throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, "distrlog-maxlogRecSize");
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    Utils.ioResult(writer.write(new LogRecord(1L, DLMTestUtil.repeatString(
                            DLMTestUtil.repeatString("abcdefgh", 256), 512).getBytes())));
}
 

@BeforeClass
public static void setupDefaults() throws Exception {
    defaultUri = DLMTestUtil.createDLMURI(zkPort, defaultPath);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("DefaultStream", conf, defaultUri);
    bindStream(defaultUri, defaultLedgerPath, defaultHost);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 8192);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testDeleteLog() throws Exception {
    String name = "delete-log-should-delete-ledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    // Create the log and write some records
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();
    writer.closeAndComplete();
    BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
    assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid, txid - 1,
        perStreamLogWriter.getLogSegmentSequenceNumber()), false));
    Utils.ioResult(blplm.asyncClose());

    // Should be able to open the underline ledger using BK client
    long ledgerId = perStreamLogWriter.getLogSegmentId();
    BKNamespaceDriver driver = (BKNamespaceDriver) dlm.getNamespaceDriver();
    driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
        BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    // Delete the log and we shouldn't be able the open the ledger
    dlm.delete();
    try {
        driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
            BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
        fail("Should fail to open ledger after we delete the log");
    } catch (BKException.BKNoSuchLedgerExistsException e) {
        // ignore
    }
    // delete again should not throw any exception
    try {
        dlm.delete();
    } catch (IOException ioe) {
        fail("Delete log twice should not throw any exception");
    }
}
 

@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(8);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 5;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    // Every 10th record broken. Reading 8 at once, beginning from 0:
    // 1. range 0-7 will be corrupted and discarded
    // 2. range 1-8 will be good, but only contain 4 records
    // And so on for the next segment, so 4 records in each segment, for 12 good records
    for (int i = 0; i < 12; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }

    Utils.close(reader);
    dlm.close();
}
 

@Test(timeout = 60000)
public void testSimpleRead() throws Exception {
    String name = "distrlog-simpleread";
    DistributedLogManager dlm = createNewDLM(conf, name);
    final long numTransactions = 10000;
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i <= numTransactions; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();

    assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1));
    dlm.close();
}
 

@Test(timeout = 60000)
public void testCloseReaderToCancelPendingReads() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    List<CompletableFuture<List<Entry.Reader>>> futures = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
        futures.add(reader.readNext(1));
    }
    assertFalse("Reader should not be closed yet", reader.isClosed());
    Utils.close(reader);
    for (CompletableFuture<List<Entry.Reader>> future : futures) {
        try {
            Utils.ioResult(future);
            fail("The read request should be cancelled");
        } catch (ReadCancelledException rce) {
            // expected
        }
    }
    assertFalse(reader.hasCaughtUpOnInprogress());
    assertTrue("Reader should be closed yet", reader.isClosed());
}
 

static void writeRecordsForNonBlockingReads(DistributedLogConfiguration conf,
                                     DistributedLogManager dlm,
                                     boolean recover,
                                     long segmentSize)
        throws Exception {
    long txId = 1;
    for (long i = 0; i < 3; i++) {
        BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j < segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
        }
        if (recover) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
            TimeUnit.MILLISECONDS.sleep(300);
            writer.abort();
            LOG.debug("Recovering Segments");
            BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
            Utils.ioResult(blplm.recoverIncompleteLogSegments());
            Utils.ioResult(blplm.asyncClose());
            LOG.debug("Recovered Segments");
        } else {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
            writer.closeAndComplete();
        }
        TimeUnit.MILLISECONDS.sleep(300);
    }
}
 

@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    dlm.close();

    URI uri = createDLMURI("/" + name);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    Namespace nonExistentNS = NamespaceBuilder.newBuilder()
            .conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));

    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);

    namespace.close();
}
 

@Test(timeout = 60000)
public void testGetLogRecordCountWithAllControlRecords() throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, runtime.getMethodName());
    long txid = 1;
    txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 5, 0, txid);
    txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 10, 0, txid);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    CompletableFuture<Long> count = null;
    count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0));
    assertEquals(0, Utils.ioResult(count).longValue());
}
 

@Test(timeout = 60000)
public void testBasicReadAndWriteBehavior() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes();

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 11).getBytes());
    writer.write(DLMTestUtil.repeatString("abc", 40).getBytes());
    writer.force(false);
    writer.close();
    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();

    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, 23);
    assertEquals(23, read);
    read = reader.read(bytesIn, 23, 31);
    assertEquals(read, 31);
    byte[] bytesInTemp = new byte[byteStream.length];
    read = reader.read(bytesInTemp, 0, byteStream.length);
    assertEquals(read, byteStream.length - 23 - 31);
    read = new ByteArrayInputStream(bytesInTemp).read(bytesIn, 23 + 31, byteStream.length - 23 - 31);
    assertEquals(read, byteStream.length - 23 - 31);
    assertArrayEquals(bytesIn, byteStream);
    reader.close();
    dlmreader.close();
    dlmwrite.close();
}
 

AsyncLogWriter createInprogressLogSegment(DistributedLogManager dlm,
                                          DistributedLogConfiguration conf,
                                          long segmentSize) throws Exception {
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    for (long i = 1L; i <= segmentSize; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(i);
        ctrlRecord.setControl();
        Utils.ioResult(writer.write(ctrlRecord));
    }
    return writer;
}
 

static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception {
    Map<Long, LogSegmentMetadata> logSegmentMap =
            new HashMap<Long, LogSegmentMetadata>();
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    for (LogSegmentMetadata segment : segments) {
        logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment);
    }
    return logSegmentMap;
}
 

/**
 * Test Case: close the input stream
 */
@Test
public void testClose() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    DLInputStream in = new DLInputStream(dlm);
    verify(dlm, times(1)).getInputStream(eq(DLSN.InitialDLSN));
    in.close();
    verify(dlm, times(1)).close();
    verify(reader, times(1)).close();
}
 

/**
 * Create Log Segment for an pre-create stream. No max ledger sequence number recorded.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentOnPrecreatedStream() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());
    dlm.close();
    namespace.close();
}
 

public static void downloadFromBookkeeper(Namespace namespace,
                                          OutputStream outputStream,
                                          String packagePath) throws IOException {
    log.info("Downloading {} from BK...", packagePath);
    DistributedLogManager dlm = namespace.openLog(packagePath);
    try (InputStream in = new DLInputStream(dlm)) {
        int read = 0;
        byte[] bytes = new byte[1024];
        while ((read = in.read(bytes)) != -1) {
            outputStream.write(bytes, 0, read);
        }
        outputStream.flush();
    }
}