org.apache.bookkeeper.client.api.LedgerEntries Java Examples

@Test
public void testOffloadEmpty() throws Exception {
    CompletableFuture<LedgerEntries> noEntries = new CompletableFuture<>();
    noEntries.completeExceptionally(new BKException.BKReadException());

    ReadHandle readHandle = Mockito.mock(ReadHandle.class);
    Mockito.doReturn(-1L).when(readHandle).getLastAddConfirmed();
    Mockito.doReturn(noEntries).when(readHandle).readAsync(anyLong(), anyLong());
    Mockito.doReturn(0L).when(readHandle).getLength();
    Mockito.doReturn(true).when(readHandle).isClosed();
    Mockito.doReturn(1234L).when(readHandle).getId();

    UUID uuid = UUID.randomUUID();
    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler,
                                                             DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
    try {
        offloader.offload(readHandle, uuid, new HashMap<>()).get();
        Assert.fail("Shouldn't have been able to offload");
    } catch (ExecutionException e) {
        Assert.assertEquals(e.getCause().getClass(), IllegalArgumentException.class);
    }
}
 

static ReadHandle getLedgerHandle() {
    final ReadHandle lh = mock(ReadHandle.class);
    final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS);
    doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer();
    doReturn((long) 10).when(ledgerEntry).getLength();

    doAnswer((invocation) -> {
            Object[] args = invocation.getArguments();
            long firstEntry = (Long) args[0];
            long lastEntry = (Long) args[1];

            Vector<LedgerEntry> entries = new Vector<LedgerEntry>();
            for (int i = 0; i <= (lastEntry - firstEntry); i++) {
                entries.add(ledgerEntry);
            }
            LedgerEntries ledgerEntries = mock(LedgerEntries.class);
            doAnswer((invocation2) -> entries.iterator()).when(ledgerEntries).iterator();
            return CompletableFuture.completedFuture(ledgerEntries);
        }).when(lh).readAsync(anyLong(), anyLong());

    return lh;
}
 

private List<ByteBuf> readNextEntriesFromLedger(long start, long maxNumberEntries) throws IOException {
    long end = Math.min(start + maxNumberEntries - 1, ledger.getLastAddConfirmed());
    try (LedgerEntries ledgerEntriesOnce = ledger.readAsync(start, end).get()) {
        log.debug("read ledger entries. start: {}, end: {}", start, end);

        List<ByteBuf> entries = Lists.newLinkedList();

        Iterator<LedgerEntry> iterator = ledgerEntriesOnce.iterator();
        while (iterator.hasNext()) {
            LedgerEntry entry = iterator.next();
            ByteBuf buf = entry.getEntryBuffer().retain();
            int entryLength = buf.readableBytes();
            long entryId = entry.getEntryId();

            CompositeByteBuf entryBuf = PulsarByteBufAllocator.DEFAULT.compositeBuffer(2);
            ByteBuf entryHeaderBuf = PulsarByteBufAllocator.DEFAULT.buffer(ENTRY_HEADER_SIZE, ENTRY_HEADER_SIZE);

            entryHeaderBuf.writeInt(entryLength).writeLong(entryId);
            entryBuf.addComponents(true, entryHeaderBuf, buf);

            entries.add(entryBuf);
        }
        return entries;
    } catch (InterruptedException | ExecutionException e) {
        log.error("Exception when get CompletableFuture<LedgerEntries>. ", e);
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        throw new IOException(e);
    }
}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    CompletableFuture<LedgerEntries> future = new CompletableFuture<>();
    LedgerEntries entries = new MockLedgerEntries(ledgerId,
        (int)firstEntry,
        (int)(lastEntry - firstEntry + 1),
            entrySize, dataSupplier);

    future.complete(entries);
    return future;
}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    List<LedgerEntry> readEntries = Lists.newArrayList();
    for (long eid = firstEntry; eid <= lastEntry; eid++) {
        ByteBuf buf = entries.get((int)eid).retainedSlice();
        readEntries.add(LedgerEntryImpl.create(id, eid, buf.readableBytes(), buf));
    }
    return CompletableFuture.completedFuture(LedgerEntriesImpl.create(readEntries));
}
 

@Test
public void testOffloadAndRead() throws Exception {
    ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3);
    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler,
                                                             DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
    UUID uuid = UUID.randomUUID();
    offloader.offload(toWrite, uuid, new HashMap<>()).get();

    ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get();
    Assert.assertEquals(toTest.getLastAddConfirmed(), toWrite.getLastAddConfirmed());

    try (LedgerEntries toWriteEntries = toWrite.read(0, toWrite.getLastAddConfirmed());
         LedgerEntries toTestEntries = toTest.read(0, toTest.getLastAddConfirmed())) {
        Iterator<LedgerEntry> toWriteIter = toWriteEntries.iterator();
        Iterator<LedgerEntry> toTestIter = toTestEntries.iterator();

        while (toWriteIter.hasNext() && toTestIter.hasNext()) {
            LedgerEntry toWriteEntry = toWriteIter.next();
            LedgerEntry toTestEntry = toTestIter.next();

            Assert.assertEquals(toWriteEntry.getLedgerId(), toTestEntry.getLedgerId());
            Assert.assertEquals(toWriteEntry.getEntryId(), toTestEntry.getEntryId());
            Assert.assertEquals(toWriteEntry.getLength(), toTestEntry.getLength());
            Assert.assertEquals(toWriteEntry.getEntryBuffer(), toTestEntry.getEntryBuffer());
        }
        Assert.assertFalse(toWriteIter.hasNext());
        Assert.assertFalse(toTestIter.hasNext());
    }
}
 

MockOffloadReadHandle(ReadHandle toCopy) throws Exception {
    id = toCopy.getId();
    long lac = toCopy.getLastAddConfirmed();
    try (LedgerEntries entries = toCopy.read(0, lac)) {
        for (LedgerEntry e : entries) {
            this.entries.add(e.getEntryBuffer().retainedSlice());
        }
    }
    metadata = new MockMetadata(toCopy.getLedgerMetadata());
}
 

public static FileSystemWriter create(LedgerEntries ledgerEntriesOnce, MapFile.Writer dataWriter, Semaphore semaphore,
                                      CountDownLatch countDownLatch, AtomicLong haveOffloadEntryNumber, LedgerReader ledgerReader) {
    FileSystemWriter writer = RECYCLER.get();
    writer.ledgerReader = ledgerReader;
    writer.dataWriter = dataWriter;
    writer.countDownLatch = countDownLatch;
    writer.haveOffloadEntryNumber = haveOffloadEntryNumber;
    writer.ledgerEntriesOnce = ledgerEntriesOnce;
    writer.semaphore = semaphore;
    return writer;
}
 

public static void scanRawLedger(long ledgerId, long fromId, long toId, herddb.client.ClientConfiguration clientConfiguration,
        ZookeeperMetadataStorageManager metadataStorageManager, Consumer<LogEntryWithSequenceNumber> consumer) throws Exception {
    ClientConfiguration config = new ClientConfiguration();
    config.setZkServers(metadataStorageManager.getZkAddress());
    config.setZkTimeout(metadataStorageManager.getZkSessionTimeout());
    config.setZkLedgersRootPath(clientConfiguration.getString(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH,
            ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_PATH_DEFAULT));
    config.setEnableParallelRecoveryRead(true);
    config.setEnableDigestTypeAutodetection(true);

    try (org.apache.bookkeeper.client.api.BookKeeper bookKeeper = org.apache.bookkeeper.client.api.BookKeeper.newBuilder(config).build();) {
        try (ReadHandle lh = bookKeeper
                .newOpenLedgerOp()
                .withRecovery(false)
                .withLedgerId(ledgerId)
                .withPassword(BookkeeperCommitLog.SHARED_SECRET.getBytes(StandardCharsets.UTF_8))
                .execute()
                .get()) {
            long lastAddConfirmed = lh.readLastAddConfirmed();
            if (toId < 0) {
                toId = lastAddConfirmed;
            }
            LOG.log(Level.INFO, "Scanning Ledger {0} from {1} to {2} LAC {3}", new Object[]{ledgerId, fromId, toId, lastAddConfirmed});
            for (long id = fromId; id <= toId; id++) {
                try (LedgerEntries entries = lh.readUnconfirmed(id, id);) {
                    LedgerEntry entry = entries.getEntry(id);
                    LogEntry lEntry = LogEntry.deserialize(entry.getEntryBytes());
                    LogEntryWithSequenceNumber e = new LogEntryWithSequenceNumber(
                            new LogSequenceNumber(ledgerId, id),
                            lEntry);
                    consumer.accept(e);
                }
            }
        }
    }

}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    return bk.getProgrammedFailure().thenComposeAsync((res) -> {
            log.debug("readEntries: first={} last={} total={}", firstEntry, lastEntry, entries.size());
            List<LedgerEntry> seq = new ArrayList<>();
            long entryId = firstEntry;
            while (entryId <= lastEntry && entryId < entries.size()) {
                seq.add(entries.get((int) entryId++).duplicate());
            }
            log.debug("Entries read: {}", seq);

            return FutureUtils.value(LedgerEntriesImpl.create(seq));
        });
}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    if (log.isDebugEnabled()) {
        log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry);
    }
    CompletableFuture<LedgerEntries> promise = new CompletableFuture<>();
    executor.submit(() -> {
        if (firstEntry > lastEntry
                || firstEntry < 0
                || lastEntry > getLastAddConfirmed()) {
            promise.completeExceptionally(new BKException.BKIncorrectParameterException());
            return;
        }
        long entriesToRead = (lastEntry - firstEntry) + 1;
        List<LedgerEntry> entries = new ArrayList<LedgerEntry>();
        long nextExpectedId = firstEntry;
        LongWritable key = new LongWritable();
        BytesWritable value = new BytesWritable();
        try {
            key.set(nextExpectedId - 1);
            reader.seek(key);
            while (entriesToRead > 0) {
                reader.next(key, value);
                int length = value.getLength();
                long entryId = key.get();
                if (entryId == nextExpectedId) {
                    ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(length, length);
                    entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf));
                    buf.writeBytes(value.copyBytes());
                    entriesToRead--;
                    nextExpectedId++;
                } else if (entryId > lastEntry) {
                    log.info("Expected to read {}, but read {}, which is greater than last entry {}",
                            nextExpectedId, entryId, lastEntry);
                    throw new BKException.BKUnexpectedConditionException();
                }
        }
            promise.complete(LedgerEntriesImpl.create(entries));
        } catch (Throwable t) {
            promise.completeExceptionally(t);
            entries.forEach(LedgerEntry::close);
        }
    });
    return promise;
}
 

@Test
void whenWriteAndReadEntriesAsync_thenSuccess() throws Exception {
    CompletableFuture<Long> f = bk.newCreateLedgerOp()
      .withDigestType(DigestType.MAC)
      .withPassword(ledgerPassword)
      .execute()
      .thenApply((wh) -> {
          List<CompletableFuture<Long>> ops = new ArrayList<>();
          for (int i = 0; i < 1000; i++) {
              byte[] data = String.format("message-%04d", i)
                .getBytes();
              ops.add(wh.appendAsync(data));
          }
          return CompletableFuture.allOf(ops.stream()
            .toArray(CompletableFuture[]::new))
            .thenCompose((v) -> wh.closeAsync())
            .thenApply((v) -> wh.getId());
      })
      .thenCompose((lf) -> lf); // flatten the futures
    Long ledgerId = f.get(5, TimeUnit.MINUTES);
    LOG.info("Ledger created with 1000 entries: ledgerId=" + ledgerId);

    // Now let's read data back...
    CompletableFuture<LedgerEntries> ef = bk.newOpenLedgerOp()
      .withLedgerId(ledgerId)
      .withPassword(ledgerPassword)
      .withDigestType(DigestType.MAC)
      .execute()
      .thenCompose((rh) -> {
          return rh.readLastAddConfirmedAsync()
            .thenCompose((lastId) -> rh.readAsync(0, lastId));
      });
    LedgerEntries entries = ef.get(5, TimeUnit.MINUTES);
    
    // Check all writes where OK
    long count = 0;
    Iterator<org.apache.bookkeeper.client.api.LedgerEntry> it = entries.iterator();
    while (it.hasNext()) {
        org.apache.bookkeeper.client.api.LedgerEntry e = it.next();
        String msg = new String(e.getEntryBytes());
        assertEquals(String.format("message-%04d", count), msg);
        count++;
    }
    assertEquals(1000, count);
    LOG.info("Got entries: count=" + count);
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return unsupported();
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return readAsync(firstEntry, lastEntry);
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return readHandle.readUnconfirmedAsync(firstEntry, lastEntry);
}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    return readHandle.readAsync(firstEntry, lastEntry);
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return readAsync(firstEntry, lastEntry);
}
 

@Override
public void run() {
    if (readHandle.getLength() == 0 || !readHandle.isClosed() || readHandle.getLastAddConfirmed() < 0) {
        promise.completeExceptionally(
                new IllegalArgumentException("An empty or open ledger should never be offloaded"));
        return;
    }
    long ledgerId = readHandle.getId();
    String storagePath = getStoragePath(storageBasePath, extraMetadata.get(MANAGED_LEDGER_NAME));
    String dataFilePath = getDataFilePath(storagePath, ledgerId, uuid);
    LongWritable key = new LongWritable();
    BytesWritable value = new BytesWritable();
    try {
        MapFile.Writer dataWriter = new MapFile.Writer(configuration,
                new Path(dataFilePath),
                MapFile.Writer.keyClass(LongWritable.class),
                MapFile.Writer.valueClass(BytesWritable.class));
        //store the ledgerMetadata in -1 index
        key.set(METADATA_KEY_INDEX);
        byte[] ledgerMetadata = buildLedgerMetadataFormat(readHandle.getLedgerMetadata());
        value.set(ledgerMetadata, 0, ledgerMetadata.length);
        dataWriter.append(key, value);
        AtomicLong haveOffloadEntryNumber = new AtomicLong(0);
        long needToOffloadFirstEntryNumber = 0;
        CountDownLatch countDownLatch;
        //avoid prefetch too much data into memory
        Semaphore semaphore = new Semaphore(managedLedgerOffloadPrefetchRounds);
        do {
            long end = Math.min(needToOffloadFirstEntryNumber + ENTRIES_PER_READ - 1, readHandle.getLastAddConfirmed());
            log.debug("read ledger entries. start: {}, end: {}", needToOffloadFirstEntryNumber, end);
            LedgerEntries ledgerEntriesOnce = readHandle.readAsync(needToOffloadFirstEntryNumber, end).get();
            semaphore.acquire();
            countDownLatch = new CountDownLatch(1);
            assignmentScheduler.chooseThread(ledgerId).submit(FileSystemWriter.create(ledgerEntriesOnce, dataWriter, semaphore,
                    countDownLatch, haveOffloadEntryNumber, this)).addListener(() -> {
            }, Executors.newSingleThreadExecutor());
            needToOffloadFirstEntryNumber = end + 1;
        } while (needToOffloadFirstEntryNumber - 1 != readHandle.getLastAddConfirmed() && fileSystemWriteException == null);
        countDownLatch.await();
        if (fileSystemWriteException != null) {
            throw fileSystemWriteException;
        }
        IOUtils.closeStream(dataWriter);
        promise.complete(null);
    } catch (Exception e) {
        log.error("Exception when get CompletableFuture<LedgerEntries> : ManagerLedgerName: {}, " +
                "LedgerId: {}, UUID: {} ", extraMetadata.get(MANAGED_LEDGER_NAME), ledgerId, uuid, e);
        if (e instanceof InterruptedException) {
            Thread.currentThread().interrupt();
        }
        promise.completeExceptionally(e);
    }
}
 

@Test
public void testOffloadReadRandomAccess() throws Exception {
    ReadHandle toWrite = buildReadHandle(DEFAULT_BLOCK_SIZE, 3);
    long[][] randomAccesses = new long[10][2];
    Random r = new Random(0);
    for (int i = 0; i < 10; i++) {
        long first = r.nextInt((int)toWrite.getLastAddConfirmed());
        long second = r.nextInt((int)toWrite.getLastAddConfirmed());
        if (second < first) {
            long tmp = first;
            first = second;
            second = tmp;
        }
        randomAccesses[i][0] = first;
        randomAccesses[i][1] = second;
    }

    LedgerOffloader offloader = new BlobStoreManagedLedgerOffloader(blobStore, BUCKET, scheduler,
                                                             DEFAULT_BLOCK_SIZE, DEFAULT_READ_BUFFER_SIZE);
    UUID uuid = UUID.randomUUID();
    offloader.offload(toWrite, uuid, new HashMap<>()).get();

    ReadHandle toTest = offloader.readOffloaded(toWrite.getId(), uuid, Collections.emptyMap()).get();
    Assert.assertEquals(toTest.getLastAddConfirmed(), toWrite.getLastAddConfirmed());

    for (long[] access : randomAccesses) {
        try (LedgerEntries toWriteEntries = toWrite.read(access[0], access[1]);
             LedgerEntries toTestEntries = toTest.read(access[0], access[1])) {
            Iterator<LedgerEntry> toWriteIter = toWriteEntries.iterator();
            Iterator<LedgerEntry> toTestIter = toTestEntries.iterator();

            while (toWriteIter.hasNext() && toTestIter.hasNext()) {
                LedgerEntry toWriteEntry = toWriteIter.next();
                LedgerEntry toTestEntry = toTestIter.next();

                Assert.assertEquals(toWriteEntry.getLedgerId(), toTestEntry.getLedgerId());
                Assert.assertEquals(toWriteEntry.getEntryId(), toTestEntry.getEntryId());
                Assert.assertEquals(toWriteEntry.getLength(), toTestEntry.getLength());
                Assert.assertEquals(toWriteEntry.getEntryBuffer(), toTestEntry.getEntryBuffer());
            }
            Assert.assertFalse(toWriteIter.hasNext());
            Assert.assertFalse(toTestIter.hasNext());
        }
    }
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return readAsync(firstEntry, lastEntry);
}
 

@Override
public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
    return readAsync(firstEntry, lastEntry);
}
 

@Override
public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
    log.debug("Ledger {}: reading {} - {}", getId(), firstEntry, lastEntry);
    CompletableFuture<LedgerEntries> promise = new CompletableFuture<>();
    executor.submit(() -> {
            if (firstEntry > lastEntry
                || firstEntry < 0
                || lastEntry > getLastAddConfirmed()) {
                promise.completeExceptionally(new BKException.BKIncorrectParameterException());
                return;
            }
            long entriesToRead = (lastEntry - firstEntry) + 1;
            List<LedgerEntry> entries = new ArrayList<LedgerEntry>();
            long nextExpectedId = firstEntry;
            try {
                while (entriesToRead > 0) {
                    int length = dataStream.readInt();
                    if (length < 0) { // hit padding or new block
                        inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset());
                        continue;
                    }
                    long entryId = dataStream.readLong();

                    if (entryId == nextExpectedId) {
                        ByteBuf buf = PulsarByteBufAllocator.DEFAULT.buffer(length, length);
                        entries.add(LedgerEntryImpl.create(ledgerId, entryId, length, buf));
                        int toWrite = length;
                        while (toWrite > 0) {
                            toWrite -= buf.writeBytes(dataStream, toWrite);
                        }
                        entriesToRead--;
                        nextExpectedId++;
                    } else if (entryId > nextExpectedId) {
                        inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset());
                        continue;
                    } else if (entryId < nextExpectedId
                               && !index.getIndexEntryForEntry(nextExpectedId).equals(
                                       index.getIndexEntryForEntry(entryId)))  {
                        inputStream.seek(index.getIndexEntryForEntry(nextExpectedId).getDataOffset());
                        continue;
                    } else if (entryId > lastEntry) {
                        log.info("Expected to read {}, but read {}, which is greater than last entry {}",
                                 nextExpectedId, entryId, lastEntry);
                        throw new BKException.BKUnexpectedConditionException();
                    } else {
                        inputStream.skip(length);
                    }
                }

                promise.complete(LedgerEntriesImpl.create(entries));
            } catch (Throwable t) {
                promise.completeExceptionally(t);
                entries.forEach(LedgerEntry::close);
            }
        });
    return promise;
}