org.apache.distributedlog.api.DistributedLogManager Java Examples

The following examples show how to use org.apache.distributedlog.api.DistributedLogManager. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestNonBlockingReads.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 15000)
public void testHandleInconsistentMetadataNonBlocking() throws Exception {
    String name = "distrlog-inconsistent-metadata-nonblocking-read";
    long numRecordsWritten = createStreamWithInconsistentMetadata(name);

    DistributedLogManager dlm = createNewDLM(conf, name);
    try {
        LogReader reader = dlm.getInputStream(45);
        long numRecordsRead = 0;
        long lastTxId = -1;
        while (numRecordsRead < (numRecordsWritten / 2)) {
            LogRecord record = reader.readNext(false);
            if (record != null) {
                DLMTestUtil.verifyLogRecord(record);
                Assert.assertTrue(lastTxId < record.getTransactionId());
                lastTxId = record.getTransactionId();
                numRecordsRead++;
            } else {
                Thread.sleep(1);
            }
        }
        reader.close();
    } finally {
        dlm.close();
    }
}
 
Example #2
Source File: TestAsyncReaderLock.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testReaderLockSharedDlmDoesNotConflict() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogManager dlm0 = createNewDLM(conf, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));
    writer.write(DLMTestUtil.getLogRecordInstance(2L));
    writer.closeAndComplete();

    DistributedLogManager dlm1 = createNewDLM(conf, name);
    CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);
    CompletableFuture<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN);

    // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the
    // system currently works.
    Utils.ioResult(futureReader1);
    Utils.ioResult(futureReader2);

    dlm0.close();
    dlm1.close();
}
 
Example #3
Source File: TestAppendOnlyStreamWriter.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration();
    conf.setPeriodicFlushFrequencyMilliSeconds(1 * 1000);
    conf.setImmediateFlushEnabled(false);
    conf.setOutputBufferSize(1024 * 1024);

    DistributedLogManager dlmwriter = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes();

    AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter();
    assertEquals(0, writer.position());

    Utils.ioResult(writer.write(byteStream));
    Thread.sleep(100); // let WriteCompleteListener have time to run
    assertEquals(33, writer.position());

    writer.close();
    dlmwriter.close();
}
 
Example #4
Source File: DLAuditor.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private List<Long> collectLedgersFromStream(Namespace namespace,
                                            String stream,
                                            Set<Long> ledgers)
        throws IOException {
    DistributedLogManager dlm = namespace.openLog(stream);
    try {
        List<LogSegmentMetadata> segments = dlm.getLogSegments();
        List<Long> sLedgers = new ArrayList<Long>();
        for (LogSegmentMetadata segment : segments) {
            synchronized (ledgers) {
                ledgers.add(segment.getLogSegmentId());
            }
            sLedgers.add(segment.getLogSegmentId());
        }
        return sLedgers;
    } finally {
        dlm.close();
    }
}
 
Example #5
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    DistributedLogManager dlm = getNamespace().openLog(getStreamName());
    try {
        long count = 0;
        if (null == endDLSN) {
            count = countToLastRecord(dlm);
        } else {
            count = countFromStartToEnd(dlm);
        }
        System.out.println("total is " + count + " records.");
        return 0;
    } finally {
        dlm.close();
    }
}
 
Example #6
Source File: DLFileSystem.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
public FSDataInputStream open(Path path, int bufferSize)
        throws IOException {
    try {
        DistributedLogManager dlm = namespace.openLog(getStreamName(path));
        LogReader reader;
        try {
            reader = dlm.openLogReader(DLSN.InitialDLSN);
        } catch (LogNotFoundException lnfe) {
            throw new FileNotFoundException(path.toString());
        } catch (LogEmptyException lee) {
            throw new FileNotFoundException(path.toString());
        }
        return new FSDataInputStream(
            new BufferedFSInputStream(
                new DLInputStream(dlm, reader, 0L),
                bufferSize));
    } catch (LogNotFoundException e) {
        throw new FileNotFoundException(path.toString());
    }
}
 
Example #7
Source File: TestReadAheadEntryReader.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
void generateCompletedLogSegments(DistributedLogManager dlm,
                                  long numCompletedSegments,
                                  long segmentSize,
                                  long startTxId) throws Exception {

    long txid = startTxId;
    for (long i = 0; i < numCompletedSegments; i++) {
        AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
        for (long j = 1; j <= segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++)));
            LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(txid);
            ctrlRecord.setControl();
            Utils.ioResult(writer.write(ctrlRecord));
        }
        Utils.close(writer);
    }
}
 
Example #8
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
private void truncateStreams(Namespace namespace, List<String> streams,
                             int tid, int numStreamsPerThreads) throws IOException {
    int startIdx = tid * numStreamsPerThreads;
    int endIdx = Math.min(streams.size(), (tid + 1) * numStreamsPerThreads);
    for (int i = startIdx; i < endIdx; i++) {
        String s = streams.get(i);
        DistributedLogManager dlm = namespace.openLog(s);
        try {
            if (deleteStream) {
                dlm.delete();
            } else {
                dlm.purgeLogsOlderThan(Long.MAX_VALUE);
            }
        } finally {
            dlm.close();
        }
    }
}
 
Example #9
Source File: DlogUploaderTest.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
@Test
public void testUploadPackageExisting() throws Exception {
  uploader = Mockito.spy(uploader);

  Namespace ns = mock(Namespace.class);
  when(nsBuilder.build()).thenReturn(ns);
  when(ns.logExists(anyString())).thenReturn(true);
  DistributedLogManager dlm = mock(DistributedLogManager.class);
  when(ns.openLog(anyString())).thenReturn(dlm);
  AppendOnlyStreamWriter asw = mock(AppendOnlyStreamWriter.class);
  when(dlm.getAppendOnlyStreamWriter()).thenReturn(asw);

  Mockito.doReturn(true).when(uploader).isLocalFileExists(Mockito.anyString());

  uploader.initialize(config);
  uploader.uploadPackage();

  verify(ns, times(1)).deleteLog(eq(uploader.getPackageName()));
  verify(copier, times(1))
      .copyFileToStream(eq(uploader.getTopologyPackageLocation()), any(OutputStream.class));
  verify(asw, times(1)).close();
  verify(dlm, times(1)).close();
}
 
Example #10
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 60000)
public void testTwoWritersOnLockDisabled() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setOutputBufferSize(0);
    confLocal.setWriteLockEnabled(false);
    String name = "distrlog-two-writers-lock-disabled";
    DistributedLogManager manager = createNewDLM(confLocal, name);
    AsyncLogWriter writer1 = Utils.ioResult(manager.openAsyncLogWriter());
    Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(1L)));
    AsyncLogWriter writer2 = Utils.ioResult(manager.openAsyncLogWriter());
    Utils.ioResult(writer2.write(DLMTestUtil.getLogRecordInstance(2L)));

    // write a record to writer 1 again
    try {
        Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(3L)));
        fail("Should fail writing record to writer 1 again as writer 2 took over the ownership");
    } catch (BKTransmitException bkte) {
        assertEquals(BKException.Code.LedgerFencedException, bkte.getBKResultCode());
    }
}
 
Example #11
Source File: DlogStorageTest.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
@Test
public void testStore() throws Exception {
  PowerMockito.mockStatic(CheckpointManager.InstanceStateCheckpoint.class);
  CheckpointManager.InstanceStateCheckpoint mockCheckpointState =
      mock(CheckpointManager.InstanceStateCheckpoint.class);

  final CheckpointInfo info = new CheckpointInfo(
      StatefulStorageTestContext.CHECKPOINT_ID, instance);
  Checkpoint checkpoint = new Checkpoint(mockCheckpointState);

  DistributedLogManager mockDLM = mock(DistributedLogManager.class);
  when(mockNamespace.openLog(anyString())).thenReturn(mockDLM);
  AppendOnlyStreamWriter mockWriter = mock(AppendOnlyStreamWriter.class);
  when(mockDLM.getAppendOnlyStreamWriter()).thenReturn(mockWriter);

  dlogStorage.storeCheckpoint(info, checkpoint);

  verify(mockWriter).markEndOfStream();
  verify(mockWriter).close();
}
 
Example #12
Source File: TestReader.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
public TestReader(String name,
                  DistributedLogManager dlm,
                  DLSN startDLSN,
                  boolean simulateErrors,
                  int delayMs,
                  CountDownLatch readyLatch,
                  CountDownLatch countLatch,
                  CountDownLatch completionLatch) {
    this.readerName = name;
    this.dlm = dlm;
    this.startDLSN = startDLSN;
    this.simulateErrors = simulateErrors;
    this.delayMs = delayMs;
    this.readyLatch = readyLatch;
    this.countLatch = countLatch;
    this.completionLatch = completionLatch;
    // States
    this.errorsFound = new AtomicBoolean(false);
    this.readCount = new AtomicInteger(0);
    this.positionReaderCount = new AtomicInteger(0);
    // Executors
    this.executorService = Executors.newSingleThreadScheduledExecutor();
}
 
Example #13
Source File: TestAsyncReaderWriter.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
/**
 * Write records into <i>numLogSegments</i> log segments.
 * Each log segment has <i>numRecordsPerLogSegment</i> records.
 *
 * @param dlm
 *          distributedlog manager
 * @param numLogSegments
 *          number of log segments
 * @param numRecordsPerLogSegment
 *          number records per log segment
 * @param startTxId
 *          start tx id
 * @return next tx id
 */
private static long writeRecords(DistributedLogManager dlm,
                                 int numLogSegments,
                                 int numRecordsPerLogSegment,
                                 long startTxId,
                                 boolean emptyRecord) throws IOException {
    long txid = startTxId;
    for (long i = 0; i < numLogSegments; i++) {
        BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        for (long j = 1; j <= numRecordsPerLogSegment; j++) {
            if (emptyRecord) {
                writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++));
            } else {
                writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++));
            }
        }
        writer.closeAndComplete();
    }
    return txid;
}
 
Example #14
Source File: TestAsyncBulkWrite.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
/**
 * Test Case: A large write batch will span multiple packets.
 * @throws Exception
 */
@Test(timeout = 60000)
public void testAsyncBulkWriteSpanningPackets() throws Exception {
    String name = "distrlog-testAsyncBulkWriteSpanningPackets";
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);
    confLocal.setOutputBufferSize(1024);
    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned());

    // First entry.
    int numTransmissions = 4;
    int recSize = 10 * 1024;
    int batchSize = (numTransmissions * MAX_LOGRECORDSET_SIZE + 1) / recSize;
    long ledgerIndex = 1;
    long entryIndex = 0;
    long slotIndex = 0;
    long txIndex = 1;
    DLSN dlsn = checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex);
    assertEquals(4, dlsn.getEntryId());
    assertEquals(1, dlsn.getLogSegmentSequenceNo());

    writer.closeAndComplete();
    dlm.close();
}
 
Example #15
Source File: TestAsyncReaderWriter.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testIdleReaderExceptionWhenKeepAliveIsDisabled() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(testConf);
    confLocal.setOutputBufferSize(0);
    confLocal.setImmediateFlushEnabled(false);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setPeriodicKeepAliveMilliSeconds(0);
    confLocal.setReadLACLongPollTimeout(9);
    confLocal.setReaderIdleWarnThresholdMillis(20);
    confLocal.setReaderIdleErrorThresholdMillis(40);

    URI uri = createDLMURI("/" + name);
    ensureURICreated(uri);

    DistributedLogManager dlm = createNewDLM(confLocal, name);
    BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter());
    writer.write(DLMTestUtil.getLogRecordInstance(1L));

    AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN));
    try {
        Utils.ioResult(reader.readNext());
        fail("Should fail when stream is idle");
    } catch (IdleReaderException ire) {
        // expected
    }
    Utils.close(reader);
    writer.close();
    dlm.close();
}
 
Example #16
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000, expected = LogRecordTooLongException.class)
public void testMaxLogRecSize() throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, "distrlog-maxlogRecSize");
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    Utils.ioResult(writer.write(new LogRecord(1L, DLMTestUtil.repeatString(
                            DLMTestUtil.repeatString("abcdefgh", 256), 512).getBytes())));
}
 
Example #17
Source File: TestDistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupDefaults() throws Exception {
    defaultUri = DLMTestUtil.createDLMURI(zkPort, defaultPath);
    DistributedLogManager dlm = DLMTestUtil.createNewDLM("DefaultStream", conf, defaultUri);
    bindStream(defaultUri, defaultLedgerPath, defaultHost);
    DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 8192);
    dlm.close();
}
 
Example #18
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testDeleteLog() throws Exception {
    String name = "delete-log-should-delete-ledgers";
    DistributedLogManager dlm = createNewDLM(conf, name);
    long txid = 1;
    // Create the log and write some records
    BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter();
    writer.closeAndComplete();
    BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
    assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid, txid - 1,
        perStreamLogWriter.getLogSegmentSequenceNumber()), false));
    Utils.ioResult(blplm.asyncClose());

    // Should be able to open the underline ledger using BK client
    long ledgerId = perStreamLogWriter.getLogSegmentId();
    BKNamespaceDriver driver = (BKNamespaceDriver) dlm.getNamespaceDriver();
    driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
        BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
    // Delete the log and we shouldn't be able the open the ledger
    dlm.delete();
    try {
        driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId,
            BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8));
        fail("Should fail to open ledger after we delete the log");
    } catch (BKException.BKNoSuchLedgerExistsException e) {
        // ignore
    }
    // delete again should not throw any exception
    try {
        dlm.delete();
    } catch (IOException ioe) {
        fail("Delete log twice should not throw any exception");
    }
}
 
Example #19
Source File: TestAsyncReaderWriter.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception {
    String name = runtime.getMethodName();
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.loadConf(testConf);

    confLocal.setOutputBufferSize(0);
    confLocal.setPeriodicFlushFrequencyMilliSeconds(0);
    confLocal.setImmediateFlushEnabled(true);
    confLocal.setReadAheadWaitTime(10);
    confLocal.setReadAheadBatchSize(8);
    confLocal.setPositionGapDetectionEnabled(false);
    confLocal.setReadAheadSkipBrokenEntries(true);
    confLocal.setEIInjectReadAheadBrokenEntries(true);
    DistributedLogManager dlm = createNewDLM(confLocal, name);

    int numLogSegments = 3;
    int numRecordsPerLogSegment = 5;

    long txid = 1L;
    txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false);

    AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN);

    // Every 10th record broken. Reading 8 at once, beginning from 0:
    // 1. range 0-7 will be corrupted and discarded
    // 2. range 1-8 will be good, but only contain 4 records
    // And so on for the next segment, so 4 records in each segment, for 12 good records
    for (int i = 0; i < 12; i++) {
        LogRecordWithDLSN record = Utils.ioResult(reader.readNext());
        assertFalse(record.getDlsn().getEntryId() % 10 == 0);
    }

    Utils.close(reader);
    dlm.close();
}
 
Example #20
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testSimpleRead() throws Exception {
    String name = "distrlog-simpleread";
    DistributedLogManager dlm = createNewDLM(conf, name);
    final long numTransactions = 10000;
    BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
    for (long i = 1; i <= numTransactions; i++) {
        LogRecord op = DLMTestUtil.getLogRecordInstance(i);
        out.write(op);
    }
    out.closeAndComplete();

    assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1));
    dlm.close();
}
 
Example #21
Source File: TestBKLogSegmentEntryReader.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testCloseReaderToCancelPendingReads() throws Exception {
    DistributedLogConfiguration confLocal = new DistributedLogConfiguration();
    confLocal.addConfiguration(conf);
    confLocal.setNumPrefetchEntriesPerLogSegment(10);
    confLocal.setMaxPrefetchEntriesPerLogSegment(10);
    DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName());
    DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 1, 20);
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    assertEquals(segments.size() + " log segments found, expected to be only one",
            1, segments.size());

    BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal);
    List<CompletableFuture<List<Entry.Reader>>> futures = Lists.newArrayList();
    for (int i = 0; i < 5; i++) {
        futures.add(reader.readNext(1));
    }
    assertFalse("Reader should not be closed yet", reader.isClosed());
    Utils.close(reader);
    for (CompletableFuture<List<Entry.Reader>> future : futures) {
        try {
            Utils.ioResult(future);
            fail("The read request should be cancelled");
        } catch (ReadCancelledException rce) {
            // expected
        }
    }
    assertFalse(reader.hasCaughtUpOnInprogress());
    assertTrue("Reader should be closed yet", reader.isClosed());
}
 
Example #22
Source File: NonBlockingReadsTestUtil.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
static void writeRecordsForNonBlockingReads(DistributedLogConfiguration conf,
                                     DistributedLogManager dlm,
                                     boolean recover,
                                     long segmentSize)
        throws Exception {
    long txId = 1;
    for (long i = 0; i < 3; i++) {
        BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned();
        for (long j = 1; j < segmentSize; j++) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
        }
        if (recover) {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
            TimeUnit.MILLISECONDS.sleep(300);
            writer.abort();
            LOG.debug("Recovering Segments");
            BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
            Utils.ioResult(blplm.recoverIncompleteLogSegments());
            Utils.ioResult(blplm.asyncClose());
            LOG.debug("Recovered Segments");
        } else {
            Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++)));
            writer.closeAndComplete();
        }
        TimeUnit.MILLISECONDS.sleep(300);
    }
}
 
Example #23
Source File: TestBKDistributedLogManager.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testCheckLogExists() throws Exception {
    String name = "distrlog-check-log-exists";
    DistributedLogManager dlm = createNewDLM(conf, name);

    long txid = 1;
    LogWriter writer = dlm.startLogSegmentNonPartitioned();
    for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) {
        writer.write(DLMTestUtil.getLogRecordInstance(txid++));
    }
    writer.flush();
    writer.commit();
    writer.close();
    dlm.close();

    URI uri = createDLMURI("/" + name);
    Namespace namespace = NamespaceBuilder.newBuilder()
            .conf(conf).uri(uri).build();
    assertTrue(namespace.logExists(name));
    assertFalse(namespace.logExists("non-existent-log"));
    URI nonExistentUri = createDLMURI("/" + "non-existent-ns");
    Namespace nonExistentNS = NamespaceBuilder.newBuilder()
            .conf(conf).uri(nonExistentUri).build();
    assertFalse(nonExistentNS.logExists(name));

    int logCount = 0;
    Iterator<String> logIter = namespace.getLogs();
    while (logIter.hasNext()) {
        String log = logIter.next();
        logCount++;
        assertEquals(name, log);
    }
    assertEquals(1, logCount);

    namespace.close();
}
 
Example #24
Source File: TestBKLogReadHandler.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testGetLogRecordCountWithAllControlRecords() throws Exception {
    DistributedLogManager dlm = createNewDLM(conf, runtime.getMethodName());
    long txid = 1;
    txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 5, 0, txid);
    txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 10, 0, txid);
    BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler();
    CompletableFuture<Long> count = null;
    count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0));
    assertEquals(0, Utils.ioResult(count).longValue());
}
 
Example #25
Source File: TestAppendOnlyStreamWriter.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 60000)
public void testBasicReadAndWriteBehavior() throws Exception {
    String name = testNames.getMethodName();
    DistributedLogManager dlmwrite = createNewDLM(conf, name);
    DistributedLogManager dlmreader = createNewDLM(conf, name);
    byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes();

    long txid = 1;
    AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter();
    writer.write(DLMTestUtil.repeatString("abc", 11).getBytes());
    writer.write(DLMTestUtil.repeatString("abc", 40).getBytes());
    writer.force(false);
    writer.close();
    AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader();

    byte[] bytesIn = new byte[byteStream.length];
    int read = reader.read(bytesIn, 0, 23);
    assertEquals(23, read);
    read = reader.read(bytesIn, 23, 31);
    assertEquals(read, 31);
    byte[] bytesInTemp = new byte[byteStream.length];
    read = reader.read(bytesInTemp, 0, byteStream.length);
    assertEquals(read, byteStream.length - 23 - 31);
    read = new ByteArrayInputStream(bytesInTemp).read(bytesIn, 23 + 31, byteStream.length - 23 - 31);
    assertEquals(read, byteStream.length - 23 - 31);
    assertArrayEquals(bytesIn, byteStream);
    reader.close();
    dlmreader.close();
    dlmwrite.close();
}
 
Example #26
Source File: TestReadAheadEntryReader.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
AsyncLogWriter createInprogressLogSegment(DistributedLogManager dlm,
                                          DistributedLogConfiguration conf,
                                          long segmentSize) throws Exception {
    AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter());
    for (long i = 1L; i <= segmentSize; i++) {
        Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i)));
        LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(i);
        ctrlRecord.setControl();
        Utils.ioResult(writer.write(ctrlRecord));
    }
    return writer;
}
 
Example #27
Source File: TestDLCK.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception {
    Map<Long, LogSegmentMetadata> logSegmentMap =
            new HashMap<Long, LogSegmentMetadata>();
    List<LogSegmentMetadata> segments = dlm.getLogSegments();
    for (LogSegmentMetadata segment : segments) {
        logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment);
    }
    return logSegmentMap;
}
 
Example #28
Source File: DLInputStreamTest.java    From pulsar with Apache License 2.0 5 votes vote down vote up
/**
 * Test Case: close the input stream
 */
@Test
public void testClose() throws Exception {
    DistributedLogManager dlm = mock(DistributedLogManager.class);
    LogReader reader = mock(LogReader.class);
    when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader);

    DLInputStream in = new DLInputStream(dlm);
    verify(dlm, times(1)).getInputStream(eq(DLSN.InitialDLSN));
    in.close();
    verify(dlm, times(1)).close();
    verify(reader, times(1)).close();
}
 
Example #29
Source File: TestLogSegmentsZK.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
/**
 * Create Log Segment for an pre-create stream. No max ledger sequence number recorded.
 */
@Test(timeout = 60000)
public void testCreateLogSegmentOnPrecreatedStream() throws Exception {
    URI uri = createURI();
    String streamName = testName.getMethodName();
    DistributedLogConfiguration conf = new DistributedLogConfiguration()
            .setLockTimeout(99999)
            .setOutputBufferSize(0)
            .setImmediateFlushEnabled(true)
            .setEnableLedgerAllocatorPool(true)
            .setLedgerAllocatorPoolName("test");
    Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build();

    namespace.createLog(streamName);
    MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber());
    DistributedLogManager dlm = namespace.openLog(streamName);
    final int numSegments = 3;
    for (int i = 0; i < numSegments; i++) {
        BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned();
        out.write(DLMTestUtil.getLogRecordInstance(i));
        out.closeAndComplete();
    }
    MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf);
    assertEquals(3, max2.getSequenceNumber());
    dlm.close();
    namespace.close();
}
 
Example #30
Source File: WorkerUtils.java    From pulsar with Apache License 2.0 5 votes vote down vote up
public static void downloadFromBookkeeper(Namespace namespace,
                                          OutputStream outputStream,
                                          String packagePath) throws IOException {
    log.info("Downloading {} from BK...", packagePath);
    DistributedLogManager dlm = namespace.openLog(packagePath);
    try (InputStream in = new DLInputStream(dlm)) {
        int read = 0;
        byte[] bytes = new byte[1024];
        while ((read = in.read(bytes)) != -1) {
            outputStream.write(bytes, 0, read);
        }
        outputStream.flush();
    }
}