org.apache.bookkeeper.client.api.LedgerEntries Java Examples
The following examples show how to use
org.apache.bookkeeper.client.api.LedgerEntries.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlobStoreManagedLedgerOffloaderTest.java From pulsar with Apache License 2.0 | 6 votes |
@Test public void testOffloadEmpty() throws Exception { CompletableFuture<LedgerEntries> noEntries = new CompletableFuture<>(); noEntries.completeExceptionally(new BKException.BKReadException()); ReadHandle readHandle = Mockito.mock(ReadHandle.class); Mockito.doReturn(-1L).when(readHandle).getLastAddConfirmed(); Mockito.doReturn(noEntries).when(readHandle).readAsync(anyLong(), anyLong()); Mockito.doReturn(0L).when(readHandle).getLength(); Mockito.doReturn(true).when(readHandle).isClosed(); Mockito.doReturn(1234L).when(readHandle).getId(); UUID uuid = UUID.randomUUID(); LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler, DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE); try { offloader.offload(readHandle, uuid, new HashMap<>()).get(); Assert.fail("Shouldn't have been able to offload"); } catch (ExecutionException e) { Assert.assertEquals(e.getCause().getClass(), IllegalArgumentException.class); } }
Example #2
Source File: EntryCacheTest.java From pulsar with Apache License 2.0 | 6 votes |
static ReadHandle getLedgerHandle() { final ReadHandle lh = mock(ReadHandle.class); final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS); doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer(); doReturn((long) 10).when(ledgerEntry).getLength(); doAnswer((invocation) -> { Object[] args = invocation.getArguments(); long firstEntry = (Long) args[0]; long lastEntry = (Long) args[1]; Vector<LedgerEntry> entries = new Vector<LedgerEntry>(); for (int i = 0; i <= (lastEntry - firstEntry); i++) { entries.add(ledgerEntry); } LedgerEntries ledgerEntries = mock(LedgerEntries.class); doAnswer((invocation2) -> entries.iterator()).when(ledgerEntries).iterator(); return CompletableFuture.completedFuture(ledgerEntries); }).when(lh).readAsync(anyLong(), anyLong()); return lh; }
Example #3
Source File: BlockAwareSegmentInputStreamImpl.java From pulsar with Apache License 2.0 | 5 votes |
private List<ByteBuf> readNextEntriesFromLedger(long start, long maxNumberEntries) throws IOException { long end = Math.min(start + maxNumberEntries - 1, ledger.getLastAddConfirmed()); try (LedgerEntries ledgerEntriesOnce = ledger.readAsync(start, end).get()) { log.debug("read ledger entries. start: {}, end: {}", start, end); List<ByteBuf> entries = Lists.newLinkedList(); Iterator<LedgerEntry> iterator = ledgerEntriesOnce.iterator(); while (iterator.hasNext()) { LedgerEntry entry = iterator.next(); ByteBuf buf = entry.getEntryBuffer().retain(); int entryLength = buf.readableBytes(); long entryId = entry.getEntryId(); CompositeByteBuf entryBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer(2); ByteBuf entryHeaderBuf = PulsarByteBufAllocator.DEFAULT.buffer(ENTRY_HEADER_SIZE, ENTRY_HEADER_SIZE); entryHeaderBuf.writeInt(entryLength).writeLong(entryId); entryBuf.addComponents(true, entryHeaderBuf, buf); entries.add(entryBuf); } return entries; } catch (InterruptedException | ExecutionException e) { log.error("Exception when get CompletableFuture<LedgerEntries>. ", e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } throw new IOException(e); } }
Example #4
Source File: BlockAwareSegmentInputStreamTest.java From pulsar with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { CompletableFuture<LedgerEntries> future = new CompletableFuture<>(); LedgerEntries entries = new MockLedgerEntries(ledgerId, (int)firstEntry, (int)(lastEntry - firstEntry + 1), entrySize, dataSupplier); future.complete(entries); return future; }
Example #5
Source File: OffloadPrefixReadTest.java From pulsar with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { List<LedgerEntry> readEntries = Lists.newArrayList(); for (long eid = firstEntry; eid <= lastEntry; eid++) { ByteBuf buf = entries.get((int)eid).retainedSlice(); readEntries.add(LedgerEntryImpl.create(id, eid, buf.readableBytes(), buf)); } return CompletableFuture.completedFuture(LedgerEntriesImpl.create(readEntries)); }
Example #6
Source File: BlobStoreManagedLedgerOffloaderTest.java From pulsar with Apache License 2.0 | 5 votes |
@Test public void testOffloadAndRead() throws Exception { ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3); LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler, DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE); UUID uuid = UUID.randomUUID(); offloader.offload(toWrite, uuid, new HashMap<>()).get(); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get(); Assert.assertEquals(toTest.getLastAddConfirmed(), toWrite.getLastAddConfirmed()); try (LedgerEntries toWriteEntries = toWrite.read(0, toWrite.getLastAddConfirmed()); LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed())) { Iterator<LedgerEntry> toWriteIter = toWriteEntries.iterator(); Iterator<LedgerEntry> toTestIter = toTestEntries.iterator(); while (toWriteIter.hasNext() && toTestIter.hasNext()) { LedgerEntry toWriteEntry = toWriteIter.next(); LedgerEntry toTestEntry = toTestIter.next(); Assert.assertEquals(toWriteEntry.getLedgerId(), toTestEntry.getLedgerId()); Assert.assertEquals(toWriteEntry.getEntryId(), toTestEntry.getEntryId()); Assert.assertEquals(toWriteEntry.getLength(), toTestEntry.getLength()); Assert.assertEquals(toWriteEntry.getEntryBuffer(), toTestEntry.getEntryBuffer()); } Assert.assertFalse(toWriteIter.hasNext()); Assert.assertFalse(toTestIter.hasNext()); } }
Example #7
Source File: OffloadPrefixReadTest.java From pulsar with Apache License 2.0 | 5 votes |
MockOffloadReadHandle(ReadHandle toCopy) throws Exception { id = toCopy.getId(); long lac = toCopy.getLastAddConfirmed(); try (LedgerEntries entries = toCopy.read(0, lac)) { for (LedgerEntry e : entries) { this.entries.add(e.getEntryBuffer().retainedSlice()); } } metadata = new MockMetadata(toCopy.getLedgerMetadata()); }
Example #8
Source File: FileSystemManagedLedgerOffloader.java From pulsar with Apache License 2.0 | 5 votes |
public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce, MapFile.Writer dataWriter, Semaphore semaphore, CountDownLatch countDownLatch, AtomicLong haveOffloadEntryNumber, LedgerReader ledgerReader) { FileSystemWriter writer = RECYCLER.get(); writer.ledgerReader = ledgerReader; writer.dataWriter = dataWriter; writer.countDownLatch = countDownLatch; writer.haveOffloadEntryNumber = haveOffloadEntryNumber; writer.ledgerEntriesOnce = ledgerEntriesOnce; writer.semaphore = semaphore; return writer; }
Example #9
Source File: BookkeeperCommitLogManager.java From herddb with Apache License 2.0 | 5 votes |
public static void scanRawLedger(long ledgerId, long fromId, long toId, herddb.client.ClientConfiguration clientConfiguration, ZookeeperMetadataStorageManager metadataStorageManager, Consumer<LogEntryWithSequenceNumber> consumer) throws Exception { ClientConfiguration config = new ClientConfiguration(); config.setZkServers(metadataStorageManager.getZkAddress()); config.setZkTimeout(metadataStorageManager.getZkSessionTimeout()); config.setZkLedgersRootPath(clientConfiguration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH, ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT)); config.setEnableParallelRecoveryRead(true); config.setEnableDigestTypeAutodetection(true); try (org.apache.bookkeeper.client.api.BookKeeper bookKeeper = org.apache.bookkeeper.client.api.BookKeeper.newBuilder(config).build();) { try (ReadHandle lh = bookKeeper .newOpenLedgerOp() .withRecovery(false) .withLedgerId(ledgerId) .withPassword(BookkeeperCommitLog.SHARED_SECRET.getBytes(StandardCharsets.UTF_8)) .execute() .get()) { long lastAddConfirmed = lh.readLastAddConfirmed(); if (toId < 0) { toId = lastAddConfirmed; } LOG.log(Level.INFO, "Scanning Ledger {0} from {1} to {2} LAC {3}", new Object[]{ledgerId, fromId, toId, lastAddConfirmed}); for (long id = fromId; id <= toId; id++) { try (LedgerEntries entries = lh.readUnconfirmed(id, id);) { LedgerEntry entry = entries.getEntry(id); LogEntry lEntry = LogEntry.deserialize(entry.getEntryBytes()); LogEntryWithSequenceNumber e = new LogEntryWithSequenceNumber( new LogSequenceNumber(ledgerId, id), lEntry); consumer.accept(e); } } } } }
Example #10
Source File: PulsarMockReadHandle.java From pulsar with Apache License 2.0 | 5 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { return bk.getProgrammedFailure().thenComposeAsync((res) -> { log.debug("readEntries: first={} last={} total={}", firstEntry, lastEntry, entries.size()); List<LedgerEntry> seq = new ArrayList<>(); long entryId = firstEntry; while (entryId <= lastEntry && entryId < entries.size()) { seq.add(entries.get((int) entryId++).duplicate()); } log.debug("Entries read: {}", seq); return FutureUtils.value(LedgerEntriesImpl.create(seq)); }); }
Example #11
Source File: FileStoreBackedReadHandleImpl.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { if (log.isDebugEnabled()) { log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); } CompletableFuture<LedgerEntries> promise = new CompletableFuture<>(); executor.submit(() -> { if (firstEntry > lastEntry || firstEntry < 0 || lastEntry > getLastAddConfirmed()) { promise.completeExceptionally(new BKException.BKIncorrectParameterException()); return; } long entriesToRead = (lastEntry - firstEntry) + 1; List<LedgerEntry> entries = new ArrayList<LedgerEntry>(); long nextExpectedId = firstEntry; LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { key.set(nextExpectedId - 1); reader.seek(key); while (entriesToRead > 0) { reader.next(key, value); int length = value.getLength(); long entryId = key.get(); if (entryId == nextExpectedId) { ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(length, length); entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf)); buf.writeBytes(value.copyBytes()); entriesToRead--; nextExpectedId++; } else if (entryId > lastEntry) { log.info("Expected to read {}, but read {}, which is greater than last entry {}", nextExpectedId, entryId, lastEntry); throw new BKException.BKUnexpectedConditionException(); } } promise.complete(LedgerEntriesImpl.create(entries)); } catch (Throwable t) { promise.completeExceptionally(t); entries.forEach(LedgerEntry::close); } }); return promise; }
Example #12
Source File: BkHelperLiveTest.java From tutorials with MIT License | 4 votes |
@Test void whenWriteAndReadEntriesAsync_thenSuccess() throws Exception { CompletableFuture<Long> f = bk.newCreateLedgerOp() .withDigestType(DigestType.MAC) .withPassword(ledgerPassword) .execute() .thenApply((wh) -> { List<CompletableFuture<Long>> ops = new ArrayList<>(); for (int i = 0; i < 1000; i++) { byte[] data = String.format("message-%04d", i) .getBytes(); ops.add(wh.appendAsync(data)); } return CompletableFuture.allOf(ops.stream() .toArray(CompletableFuture[]::new)) .thenCompose((v) -> wh.closeAsync()) .thenApply((v) -> wh.getId()); }) .thenCompose((lf) -> lf); // flatten the futures Long ledgerId = f.get(5, TimeUnit.MINUTES); LOG.info("Ledger created with 1000 entries: ledgerId=" + ledgerId); // Now let's read data back... CompletableFuture<LedgerEntries> ef = bk.newOpenLedgerOp() .withLedgerId(ledgerId) .withPassword(ledgerPassword) .withDigestType(DigestType.MAC) .execute() .thenCompose((rh) -> { return rh.readLastAddConfirmedAsync() .thenCompose((lastId) -> rh.readAsync(0, lastId)); }); LedgerEntries entries = ef.get(5, TimeUnit.MINUTES); // Check all writes where OK long count = 0; Iterator<org.apache.bookkeeper.client.api.LedgerEntry> it = entries.iterator(); while (it.hasNext()) { org.apache.bookkeeper.client.api.LedgerEntry e = it.next(); String msg = new String(e.getEntryBytes()); assertEquals(String.format("message-%04d", count), msg); count++; } assertEquals(1000, count); LOG.info("Got entries: count=" + count); }
Example #13
Source File: OffloadPrefixReadTest.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return unsupported(); }
Example #14
Source File: PulsarMockReadHandle.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return readAsync(firstEntry, lastEntry); }
Example #15
Source File: PulsarMockLedgerHandle.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return readHandle.readUnconfirmedAsync(firstEntry, lastEntry); }
Example #16
Source File: PulsarMockLedgerHandle.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { return readHandle.readAsync(firstEntry, lastEntry); }
Example #17
Source File: FileStoreBackedReadHandleImpl.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return readAsync(firstEntry, lastEntry); }
Example #18
Source File: FileSystemManagedLedgerOffloader.java From pulsar with Apache License 2.0 | 4 votes |
@Override public void run() { if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) { promise.completeExceptionally( new IllegalArgumentException("An empty or open ledger should never be offloaded")); return; } long ledgerId = readHandle.getId(); String storagePath = getStoragePath(storageBasePath, extraMetadata.get(MANAGED_LEDGER_NAME)); String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid); LongWritable key = new LongWritable(); BytesWritable value = new BytesWritable(); try { MapFile.Writer dataWriter = new MapFile.Writer(configuration, new Path(dataFilePath), MapFile.Writer.keyClass(LongWritable.class), MapFile.Writer.valueClass(BytesWritable.class)); //store the ledgerMetadata in -1 index key.set(METADATA_KEY_INDEX); byte[] ledgerMetadata = buildLedgerMetadataFormat(readHandle.getLedgerMetadata()); value.set(ledgerMetadata, 0, ledgerMetadata.length); dataWriter.append(key, value); AtomicLong haveOffloadEntryNumber = new AtomicLong(0); long needToOffloadFirstEntryNumber = 0; CountDownLatch countDownLatch; //avoid prefetch too much data into memory Semaphore semaphore = new Semaphore(managedLedgerOffloadPrefetchRounds); do { long end = Math.min(needToOffloadFirstEntryNumber + ENTRIES_PER_READ - 1, readHandle.getLastAddConfirmed()); log.debug("read ledger entries. start: {}, end: {}", needToOffloadFirstEntryNumber, end); LedgerEntries ledgerEntriesOnce = readHandle.readAsync(needToOffloadFirstEntryNumber, end).get(); semaphore.acquire(); countDownLatch = new CountDownLatch(1); assignmentScheduler.chooseThread(ledgerId).submit(FileSystemWriter.create(ledgerEntriesOnce, dataWriter, semaphore, countDownLatch, haveOffloadEntryNumber, this)).addListener(() -> { }, Executors.newSingleThreadExecutor()); needToOffloadFirstEntryNumber = end + 1; } while (needToOffloadFirstEntryNumber - 1 != readHandle.getLastAddConfirmed() && fileSystemWriteException == null); countDownLatch.await(); if (fileSystemWriteException != null) { throw fileSystemWriteException; } IOUtils.closeStream(dataWriter); promise.complete(null); } catch (Exception e) { log.error("Exception when get CompletableFuture<LedgerEntries> : ManagerLedgerName: {}, " + "LedgerId: {}, UUID: {} ", extraMetadata.get(MANAGED_LEDGER_NAME), ledgerId, uuid, e); if (e instanceof InterruptedException) { Thread.currentThread().interrupt(); } promise.completeExceptionally(e); } }
Example #19
Source File: BlobStoreManagedLedgerOffloaderTest.java From pulsar with Apache License 2.0 | 4 votes |
@Test public void testOffloadReadRandomAccess() throws Exception { ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3); long[][] randomAccesses = new long[10][2]; Random r = new Random(0); for (int i = 0; i < 10; i++) { long first = r.nextInt((int)toWrite.getLastAddConfirmed()); long second = r.nextInt((int)toWrite.getLastAddConfirmed()); if (second < first) { long tmp = first; first = second; second = tmp; } randomAccesses[i][0] = first; randomAccesses[i][1] = second; } LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler, DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE); UUID uuid = UUID.randomUUID(); offloader.offload(toWrite, uuid, new HashMap<>()).get(); ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get(); Assert.assertEquals(toTest.getLastAddConfirmed(), toWrite.getLastAddConfirmed()); for (long[] access : randomAccesses) { try (LedgerEntries toWriteEntries = toWrite.read(access[0], access[1]); LedgerEntries toTestEntries = toTest.read(access[0], access[1])) { Iterator<LedgerEntry> toWriteIter = toWriteEntries.iterator(); Iterator<LedgerEntry> toTestIter = toTestEntries.iterator(); while (toWriteIter.hasNext() && toTestIter.hasNext()) { LedgerEntry toWriteEntry = toWriteIter.next(); LedgerEntry toTestEntry = toTestIter.next(); Assert.assertEquals(toWriteEntry.getLedgerId(), toTestEntry.getLedgerId()); Assert.assertEquals(toWriteEntry.getEntryId(), toTestEntry.getEntryId()); Assert.assertEquals(toWriteEntry.getLength(), toTestEntry.getLength()); Assert.assertEquals(toWriteEntry.getEntryBuffer(), toTestEntry.getEntryBuffer()); } Assert.assertFalse(toWriteIter.hasNext()); Assert.assertFalse(toTestIter.hasNext()); } } }
Example #20
Source File: BlockAwareSegmentInputStreamTest.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return readAsync(firstEntry, lastEntry); }
Example #21
Source File: BlobStoreBackedReadHandleImpl.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) { return readAsync(firstEntry, lastEntry); }
Example #22
Source File: BlobStoreBackedReadHandleImpl.java From pulsar with Apache License 2.0 | 4 votes |
@Override public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) { log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry); CompletableFuture<LedgerEntries> promise = new CompletableFuture<>(); executor.submit(() -> { if (firstEntry > lastEntry || firstEntry < 0 || lastEntry > getLastAddConfirmed()) { promise.completeExceptionally(new BKException.BKIncorrectParameterException()); return; } long entriesToRead = (lastEntry - firstEntry) + 1; List<LedgerEntry> entries = new ArrayList<LedgerEntry>(); long nextExpectedId = firstEntry; try { while (entriesToRead > 0) { int length = dataStream.readInt(); if (length < 0) { // hit padding or new block inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); continue; } long entryId = dataStream.readLong(); if (entryId == nextExpectedId) { ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(length, length); entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf)); int toWrite = length; while (toWrite > 0) { toWrite -= buf.writeBytes(dataStream, toWrite); } entriesToRead--; nextExpectedId++; } else if (entryId > nextExpectedId) { inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); continue; } else if (entryId < nextExpectedId && !index.getIndexEntryForEntry(nextExpectedId).equals( index.getIndexEntryForEntry(entryId))) { inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset()); continue; } else if (entryId > lastEntry) { log.info("Expected to read {}, but read {}, which is greater than last entry {}", nextExpectedId, entryId, lastEntry); throw new BKException.BKUnexpectedConditionException(); } else { inputStream.skip(length); } } promise.complete(LedgerEntriesImpl.create(entries)); } catch (Throwable t) { promise.completeExceptionally(t); entries.forEach(LedgerEntry::close); } }); return promise; }