org.apache.distributedlog.api.DistributedLogManager Java Examples
The following examples show how to use
org.apache.distributedlog.api.DistributedLogManager.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestNonBlockingReads.java From distributedlog with Apache License 2.0 | 6 votes |
@Test(timeout = 15000) public void testHandleInconsistentMetadataNonBlocking() throws Exception { String name = "distrlog-inconsistent-metadata-nonblocking-read"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(45); long numRecordsRead = 0; long lastTxId = -1; while (numRecordsRead < (numRecordsWritten / 2)) { LogRecord record = reader.readNext(false); if (record != null) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } } reader.close(); } finally { dlm.close(); } }
Example #2
Source File: TestAsyncReaderLock.java From distributedlog with Apache License 2.0 | 6 votes |
@Test(timeout = 60000) public void testReaderLockSharedDlmDoesNotConflict() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); CompletableFuture<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); CompletableFuture<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the // system currently works. Utils.ioResult(futureReader1); Utils.ioResult(futureReader2); dlm0.close(); dlm1.close(); }
Example #3
Source File: TestAppendOnlyStreamWriter.java From distributedlog with Apache License 2.0 | 6 votes |
@Test(timeout = 60000) public void testPositionUpdatesOnlyAfterWriteCompletionWithoutFsync() throws Exception { String name = testNames.getMethodName(); DistributedLogConfiguration conf = new DistributedLogConfiguration(); conf.setPeriodicFlushFrequencyMilliSeconds(1 * 1000); conf.setImmediateFlushEnabled(false); conf.setOutputBufferSize(1024 * 1024); DistributedLogManager dlmwriter = createNewDLM(conf, name); byte[] byteStream = DLMTestUtil.repeatString("abc", 11).getBytes(); AppendOnlyStreamWriter writer = dlmwriter.getAppendOnlyStreamWriter(); assertEquals(0, writer.position()); Utils.ioResult(writer.write(byteStream)); Thread.sleep(100); // let WriteCompleteListener have time to run assertEquals(33, writer.position()); writer.close(); dlmwriter.close(); }
Example #4
Source File: DLAuditor.java From distributedlog with Apache License 2.0 | 6 votes |
private List<Long> collectLedgersFromStream(Namespace namespace, String stream, Set<Long> ledgers) throws IOException { DistributedLogManager dlm = namespace.openLog(stream); try { List<LogSegmentMetadata> segments = dlm.getLogSegments(); List<Long> sLedgers = new ArrayList<Long>(); for (LogSegmentMetadata segment : segments) { synchronized (ledgers) { ledgers.add(segment.getLogSegmentId()); } sLedgers.add(segment.getLogSegmentId()); } return sLedgers; } finally { dlm.close(); } }
Example #5
Source File: DistributedLogTool.java From distributedlog with Apache License 2.0 | 6 votes |
@Override protected int runCmd() throws Exception { DistributedLogManager dlm = getNamespace().openLog(getStreamName()); try { long count = 0; if (null == endDLSN) { count = countToLastRecord(dlm); } else { count = countFromStartToEnd(dlm); } System.out.println("total is " + count + " records."); return 0; } finally { dlm.close(); } }
Example #6
Source File: DLFileSystem.java From distributedlog with Apache License 2.0 | 6 votes |
@Override public FSDataInputStream open(Path path, int bufferSize) throws IOException { try { DistributedLogManager dlm = namespace.openLog(getStreamName(path)); LogReader reader; try { reader = dlm.openLogReader(DLSN.InitialDLSN); } catch (LogNotFoundException lnfe) { throw new FileNotFoundException(path.toString()); } catch (LogEmptyException lee) { throw new FileNotFoundException(path.toString()); } return new FSDataInputStream( new BufferedFSInputStream( new DLInputStream(dlm, reader, 0L), bufferSize)); } catch (LogNotFoundException e) { throw new FileNotFoundException(path.toString()); } }
Example #7
Source File: TestReadAheadEntryReader.java From distributedlog with Apache License 2.0 | 6 votes |
void generateCompletedLogSegments(DistributedLogManager dlm, long numCompletedSegments, long segmentSize, long startTxId) throws Exception { long txid = startTxId; for (long i = 0; i < numCompletedSegments; i++) { AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter()); for (long j = 1; j <= segmentSize; j++) { Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txid++))); LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(txid); ctrlRecord.setControl(); Utils.ioResult(writer.write(ctrlRecord)); } Utils.close(writer); } }
Example #8
Source File: DistributedLogTool.java From distributedlog with Apache License 2.0 | 6 votes |
private void truncateStreams(Namespace namespace, List<String> streams, int tid, int numStreamsPerThreads) throws IOException { int startIdx = tid * numStreamsPerThreads; int endIdx = Math.min(streams.size(), (tid + 1) * numStreamsPerThreads); for (int i = startIdx; i < endIdx; i++) { String s = streams.get(i); DistributedLogManager dlm = namespace.openLog(s); try { if (deleteStream) { dlm.delete(); } else { dlm.purgeLogsOlderThan(Long.MAX_VALUE); } } finally { dlm.close(); } } }
Example #9
Source File: DlogUploaderTest.java From incubator-heron with Apache License 2.0 | 6 votes |
@Test public void testUploadPackageExisting() throws Exception { uploader = Mockito.spy(uploader); Namespace ns = mock(Namespace.class); when(nsBuilder.build()).thenReturn(ns); when(ns.logExists(anyString())).thenReturn(true); DistributedLogManager dlm = mock(DistributedLogManager.class); when(ns.openLog(anyString())).thenReturn(dlm); AppendOnlyStreamWriter asw = mock(AppendOnlyStreamWriter.class); when(dlm.getAppendOnlyStreamWriter()).thenReturn(asw); Mockito.doReturn(true).when(uploader).isLocalFileExists(Mockito.anyString()); uploader.initialize(config); uploader.uploadPackage(); verify(ns, times(1)).deleteLog(eq(uploader.getPackageName())); verify(copier, times(1)) .copyFileToStream(eq(uploader.getTopologyPackageLocation()), any(OutputStream.class)); verify(asw, times(1)).close(); verify(dlm, times(1)).close(); }
Example #10
Source File: TestBKDistributedLogManager.java From distributedlog with Apache License 2.0 | 6 votes |
@Test(timeout = 60000) public void testTwoWritersOnLockDisabled() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(conf); confLocal.setOutputBufferSize(0); confLocal.setWriteLockEnabled(false); String name = "distrlog-two-writers-lock-disabled"; DistributedLogManager manager = createNewDLM(confLocal, name); AsyncLogWriter writer1 = Utils.ioResult(manager.openAsyncLogWriter()); Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(1L))); AsyncLogWriter writer2 = Utils.ioResult(manager.openAsyncLogWriter()); Utils.ioResult(writer2.write(DLMTestUtil.getLogRecordInstance(2L))); // write a record to writer 1 again try { Utils.ioResult(writer1.write(DLMTestUtil.getLogRecordInstance(3L))); fail("Should fail writing record to writer 1 again as writer 2 took over the ownership"); } catch (BKTransmitException bkte) { assertEquals(BKException.Code.LedgerFencedException, bkte.getBKResultCode()); } }
Example #11
Source File: DlogStorageTest.java From incubator-heron with Apache License 2.0 | 6 votes |
@Test public void testStore() throws Exception { PowerMockito.mockStatic(CheckpointManager.InstanceStateCheckpoint.class); CheckpointManager.InstanceStateCheckpoint mockCheckpointState = mock(CheckpointManager.InstanceStateCheckpoint.class); final CheckpointInfo info = new CheckpointInfo( StatefulStorageTestContext.CHECKPOINT_ID, instance); Checkpoint checkpoint = new Checkpoint(mockCheckpointState); DistributedLogManager mockDLM = mock(DistributedLogManager.class); when(mockNamespace.openLog(anyString())).thenReturn(mockDLM); AppendOnlyStreamWriter mockWriter = mock(AppendOnlyStreamWriter.class); when(mockDLM.getAppendOnlyStreamWriter()).thenReturn(mockWriter); dlogStorage.storeCheckpoint(info, checkpoint); verify(mockWriter).markEndOfStream(); verify(mockWriter).close(); }
Example #12
Source File: TestReader.java From distributedlog with Apache License 2.0 | 6 votes |
public TestReader(String name, DistributedLogManager dlm, DLSN startDLSN, boolean simulateErrors, int delayMs, CountDownLatch readyLatch, CountDownLatch countLatch, CountDownLatch completionLatch) { this.readerName = name; this.dlm = dlm; this.startDLSN = startDLSN; this.simulateErrors = simulateErrors; this.delayMs = delayMs; this.readyLatch = readyLatch; this.countLatch = countLatch; this.completionLatch = completionLatch; // States this.errorsFound = new AtomicBoolean(false); this.readCount = new AtomicInteger(0); this.positionReaderCount = new AtomicInteger(0); // Executors this.executorService = Executors.newSingleThreadScheduledExecutor(); }
Example #13
Source File: TestAsyncReaderWriter.java From distributedlog with Apache License 2.0 | 6 votes |
/** * Write records into <i>numLogSegments</i> log segments. * Each log segment has <i>numRecordsPerLogSegment</i> records. * * @param dlm * distributedlog manager * @param numLogSegments * number of log segments * @param numRecordsPerLogSegment * number records per log segment * @param startTxId * start tx id * @return next tx id */ private static long writeRecords(DistributedLogManager dlm, int numLogSegments, int numRecordsPerLogSegment, long startTxId, boolean emptyRecord) throws IOException { long txid = startTxId; for (long i = 0; i < numLogSegments; i++) { BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= numRecordsPerLogSegment; j++) { if (emptyRecord) { writer.write(DLMTestUtil.getEmptyLogRecordInstance(txid++)); } else { writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++)); } } writer.closeAndComplete(); } return txid; }
Example #14
Source File: TestAsyncBulkWrite.java From distributedlog with Apache License 2.0 | 6 votes |
/** * Test Case: A large write batch will span multiple packets. * @throws Exception */ @Test(timeout = 60000) public void testAsyncBulkWriteSpanningPackets() throws Exception { String name = "distrlog-testAsyncBulkWriteSpanningPackets"; DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(testConf); confLocal.setOutputBufferSize(1024); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter) (dlm.startAsyncLogSegmentNonPartitioned()); // First entry. int numTransmissions = 4; int recSize = 10 * 1024; int batchSize = (numTransmissions * MAX_LOGRECORDSET_SIZE + 1) / recSize; long ledgerIndex = 1; long entryIndex = 0; long slotIndex = 0; long txIndex = 1; DLSN dlsn = checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex); assertEquals(4, dlsn.getEntryId()); assertEquals(1, dlsn.getLogSegmentSequenceNo()); writer.closeAndComplete(); dlm.close(); }
Example #15
Source File: TestAsyncReaderWriter.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testIdleReaderExceptionWhenKeepAliveIsDisabled() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(false); confLocal.setPeriodicFlushFrequencyMilliSeconds(0); confLocal.setPeriodicKeepAliveMilliSeconds(0); confLocal.setReadLACLongPollTimeout(9); confLocal.setReaderIdleWarnThresholdMillis(20); confLocal.setReaderIdleErrorThresholdMillis(40); URI uri = createDLMURI("/" + name); ensureURICreated(uri); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter) Utils.ioResult(dlm.openAsyncLogWriter()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); AsyncLogReader reader = Utils.ioResult(dlm.openAsyncLogReader(DLSN.InitialDLSN)); try { Utils.ioResult(reader.readNext()); fail("Should fail when stream is idle"); } catch (IdleReaderException ire) { // expected } Utils.close(reader); writer.close(); dlm.close(); }
Example #16
Source File: TestBKDistributedLogManager.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000, expected = LogRecordTooLongException.class) public void testMaxLogRecSize() throws Exception { DistributedLogManager dlm = createNewDLM(conf, "distrlog-maxlogRecSize"); AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter()); Utils.ioResult(writer.write(new LogRecord(1L, DLMTestUtil.repeatString( DLMTestUtil.repeatString("abcdefgh", 256), 512).getBytes()))); }
Example #17
Source File: TestDistributedLogTool.java From distributedlog with Apache License 2.0 | 5 votes |
@BeforeClass public static void setupDefaults() throws Exception { defaultUri = DLMTestUtil.createDLMURI(zkPort, defaultPath); DistributedLogManager dlm = DLMTestUtil.createNewDLM("DefaultStream", conf, defaultUri); bindStream(defaultUri, defaultLedgerPath, defaultHost); DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 8192); dlm.close(); }
Example #18
Source File: TestBKDistributedLogManager.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testDeleteLog() throws Exception { String name = "delete-log-should-delete-ledgers"; DistributedLogManager dlm = createNewDLM(conf, name); long txid = 1; // Create the log and write some records BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter(); writer.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(txid, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); Utils.ioResult(blplm.asyncClose()); // Should be able to open the underline ledger using BK client long ledgerId = perStreamLogWriter.getLogSegmentId(); BKNamespaceDriver driver = (BKNamespaceDriver) dlm.getNamespaceDriver(); driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8)); // Delete the log and we shouldn't be able the open the ledger dlm.delete(); try { driver.getReaderBKC().get().openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, conf.getBKDigestPW().getBytes(UTF_8)); fail("Should fail to open ledger after we delete the log"); } catch (BKException.BKNoSuchLedgerExistsException e) { // ignore } // delete again should not throw any exception try { dlm.delete(); } catch (IOException ioe) { fail("Delete log twice should not throw any exception"); } }
Example #19
Source File: TestAsyncReaderWriter.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testReadBrokenEntriesAndLargeBatchSizeCrossSegment() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(testConf); confLocal.setOutputBufferSize(0); confLocal.setPeriodicFlushFrequencyMilliSeconds(0); confLocal.setImmediateFlushEnabled(true); confLocal.setReadAheadWaitTime(10); confLocal.setReadAheadBatchSize(8); confLocal.setPositionGapDetectionEnabled(false); confLocal.setReadAheadSkipBrokenEntries(true); confLocal.setEIInjectReadAheadBrokenEntries(true); DistributedLogManager dlm = createNewDLM(confLocal, name); int numLogSegments = 3; int numRecordsPerLogSegment = 5; long txid = 1L; txid = writeRecords(dlm, numLogSegments, numRecordsPerLogSegment, txid, false); AsyncLogReader reader = dlm.getAsyncLogReader(DLSN.InvalidDLSN); // Every 10th record broken. Reading 8 at once, beginning from 0: // 1. range 0-7 will be corrupted and discarded // 2. range 1-8 will be good, but only contain 4 records // And so on for the next segment, so 4 records in each segment, for 12 good records for (int i = 0; i < 12; i++) { LogRecordWithDLSN record = Utils.ioResult(reader.readNext()); assertFalse(record.getDlsn().getEntryId() % 10 == 0); } Utils.close(reader); dlm.close(); }
Example #20
Source File: TestBKDistributedLogManager.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testSimpleRead() throws Exception { String name = "distrlog-simpleread"; DistributedLogManager dlm = createNewDLM(conf, name); final long numTransactions = 10000; BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= numTransactions; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1)); dlm.close(); }
Example #21
Source File: TestBKLogSegmentEntryReader.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testCloseReaderToCancelPendingReads() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(conf); confLocal.setNumPrefetchEntriesPerLogSegment(10); confLocal.setMaxPrefetchEntriesPerLogSegment(10); DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName()); DLMTestUtil.generateCompletedLogSegments(dlm, confLocal, 1, 20); List<LogSegmentMetadata> segments = dlm.getLogSegments(); assertEquals(segments.size() + " log segments found, expected to be only one", 1, segments.size()); BKLogSegmentEntryReader reader = createEntryReader(segments.get(0), 0, confLocal); List<CompletableFuture<List<Entry.Reader>>> futures = Lists.newArrayList(); for (int i = 0; i < 5; i++) { futures.add(reader.readNext(1)); } assertFalse("Reader should not be closed yet", reader.isClosed()); Utils.close(reader); for (CompletableFuture<List<Entry.Reader>> future : futures) { try { Utils.ioResult(future); fail("The read request should be cancelled"); } catch (ReadCancelledException rce) { // expected } } assertFalse(reader.hasCaughtUpOnInprogress()); assertTrue("Reader should be closed yet", reader.isClosed()); }
Example #22
Source File: NonBlockingReadsTestUtil.java From distributedlog with Apache License 2.0 | 5 votes |
static void writeRecordsForNonBlockingReads(DistributedLogConfiguration conf, DistributedLogManager dlm, boolean recover, long segmentSize) throws Exception { long txId = 1; for (long i = 0; i < 3; i++) { BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned(); for (long j = 1; j < segmentSize; j++) { Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); } if (recover) { Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); TimeUnit.MILLISECONDS.sleep(300); writer.abort(); LOG.debug("Recovering Segments"); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); Utils.ioResult(blplm.recoverIncompleteLogSegments()); Utils.ioResult(blplm.asyncClose()); LOG.debug("Recovered Segments"); } else { Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); writer.closeAndComplete(); } TimeUnit.MILLISECONDS.sleep(300); } }
Example #23
Source File: TestBKDistributedLogManager.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testCheckLogExists() throws Exception { String name = "distrlog-check-log-exists"; DistributedLogManager dlm = createNewDLM(conf, name); long txid = 1; LogWriter writer = dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } writer.flush(); writer.commit(); writer.close(); dlm.close(); URI uri = createDLMURI("/" + name); Namespace namespace = NamespaceBuilder.newBuilder() .conf(conf).uri(uri).build(); assertTrue(namespace.logExists(name)); assertFalse(namespace.logExists("non-existent-log")); URI nonExistentUri = createDLMURI("/" + "non-existent-ns"); Namespace nonExistentNS = NamespaceBuilder.newBuilder() .conf(conf).uri(nonExistentUri).build(); assertFalse(nonExistentNS.logExists(name)); int logCount = 0; Iterator<String> logIter = namespace.getLogs(); while (logIter.hasNext()) { String log = logIter.next(); logCount++; assertEquals(name, log); } assertEquals(1, logCount); namespace.close(); }
Example #24
Source File: TestBKLogReadHandler.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testGetLogRecordCountWithAllControlRecords() throws Exception { DistributedLogManager dlm = createNewDLM(conf, runtime.getMethodName()); long txid = 1; txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 5, 0, txid); txid += DLMTestUtil.generateLogSegmentNonPartitioned(dlm, 10, 0, txid); BKLogReadHandler readHandler = ((BKDistributedLogManager) dlm).createReadHandler(); CompletableFuture<Long> count = null; count = readHandler.asyncGetLogRecordCount(new DLSN(1, 0, 0)); assertEquals(0, Utils.ioResult(count).longValue()); }
Example #25
Source File: TestAppendOnlyStreamWriter.java From distributedlog with Apache License 2.0 | 5 votes |
@Test(timeout = 60000) public void testBasicReadAndWriteBehavior() throws Exception { String name = testNames.getMethodName(); DistributedLogManager dlmwrite = createNewDLM(conf, name); DistributedLogManager dlmreader = createNewDLM(conf, name); byte[] byteStream = DLMTestUtil.repeatString("abc", 51).getBytes(); long txid = 1; AppendOnlyStreamWriter writer = dlmwrite.getAppendOnlyStreamWriter(); writer.write(DLMTestUtil.repeatString("abc", 11).getBytes()); writer.write(DLMTestUtil.repeatString("abc", 40).getBytes()); writer.force(false); writer.close(); AppendOnlyStreamReader reader = dlmreader.getAppendOnlyStreamReader(); byte[] bytesIn = new byte[byteStream.length]; int read = reader.read(bytesIn, 0, 23); assertEquals(23, read); read = reader.read(bytesIn, 23, 31); assertEquals(read, 31); byte[] bytesInTemp = new byte[byteStream.length]; read = reader.read(bytesInTemp, 0, byteStream.length); assertEquals(read, byteStream.length - 23 - 31); read = new ByteArrayInputStream(bytesInTemp).read(bytesIn, 23 + 31, byteStream.length - 23 - 31); assertEquals(read, byteStream.length - 23 - 31); assertArrayEquals(bytesIn, byteStream); reader.close(); dlmreader.close(); dlmwrite.close(); }
Example #26
Source File: TestReadAheadEntryReader.java From distributedlog with Apache License 2.0 | 5 votes |
AsyncLogWriter createInprogressLogSegment(DistributedLogManager dlm, DistributedLogConfiguration conf, long segmentSize) throws Exception { AsyncLogWriter writer = Utils.ioResult(dlm.openAsyncLogWriter()); for (long i = 1L; i <= segmentSize; i++) { Utils.ioResult(writer.write(DLMTestUtil.getLogRecordInstance(i))); LogRecord ctrlRecord = DLMTestUtil.getLogRecordInstance(i); ctrlRecord.setControl(); Utils.ioResult(writer.write(ctrlRecord)); } return writer; }
Example #27
Source File: TestDLCK.java From distributedlog with Apache License 2.0 | 5 votes |
static Map<Long, LogSegmentMetadata> getLogSegments(DistributedLogManager dlm) throws Exception { Map<Long, LogSegmentMetadata> logSegmentMap = new HashMap<Long, LogSegmentMetadata>(); List<LogSegmentMetadata> segments = dlm.getLogSegments(); for (LogSegmentMetadata segment : segments) { logSegmentMap.put(segment.getLogSegmentSequenceNumber(), segment); } return logSegmentMap; }
Example #28
Source File: DLInputStreamTest.java From pulsar with Apache License 2.0 | 5 votes |
/** * Test Case: close the input stream */ @Test public void testClose() throws Exception { DistributedLogManager dlm = mock(DistributedLogManager.class); LogReader reader = mock(LogReader.class); when(dlm.getInputStream(any(DLSN.class))).thenReturn(reader); DLInputStream in = new DLInputStream(dlm); verify(dlm, times(1)).getInputStream(eq(DLSN.InitialDLSN)); in.close(); verify(dlm, times(1)).close(); verify(reader, times(1)).close(); }
Example #29
Source File: TestLogSegmentsZK.java From distributedlog with Apache License 2.0 | 5 votes |
/** * Create Log Segment for an pre-create stream. No max ledger sequence number recorded. */ @Test(timeout = 60000) public void testCreateLogSegmentOnPrecreatedStream() throws Exception { URI uri = createURI(); String streamName = testName.getMethodName(); DistributedLogConfiguration conf = new DistributedLogConfiguration() .setLockTimeout(99999) .setOutputBufferSize(0) .setImmediateFlushEnabled(true) .setEnableLedgerAllocatorPool(true) .setLedgerAllocatorPoolName("test"); Namespace namespace = NamespaceBuilder.newBuilder().conf(conf).uri(uri).build(); namespace.createLog(streamName); MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf); assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber()); DistributedLogManager dlm = namespace.openLog(streamName); final int numSegments = 3; for (int i = 0; i < numSegments; i++) { BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); out.write(DLMTestUtil.getLogRecordInstance(i)); out.closeAndComplete(); } MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(getZooKeeperClient(namespace), uri, streamName, conf); assertEquals(3, max2.getSequenceNumber()); dlm.close(); namespace.close(); }
Example #30
Source File: WorkerUtils.java From pulsar with Apache License 2.0 | 5 votes |
public static void downloadFromBookkeeper(Namespace namespace, OutputStream outputStream, String packagePath) throws IOException { log.info("Downloading {} from BK...", packagePath); DistributedLogManager dlm = namespace.openLog(packagePath); try (InputStream in = new DLInputStream(dlm)) { int read = 0; byte[] bytes = new byte[1024]; while ((read = in.read(bytes)) != -1) { outputStream.write(bytes, 0, read); } outputStream.flush(); } }