Java Code Examples for org.apache.bookkeeper.util.IOUtils#confirmPrompt()

The following examples show how to use org.apache.bookkeeper.util.IOUtils#confirmPrompt() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    MetadataUpdater metadataUpdater = dryrun
            ? new DryrunLogSegmentMetadataStoreUpdater(getConf(),
                    getLogSegmentMetadataStore()) :
            LogSegmentMetadataStoreUpdater.createMetadataUpdater(getConf(),
                    getLogSegmentMetadataStore());
    System.out.println("List of streams : ");
    System.out.println(streams);
    if (!IOUtils.confirmPrompt("Do you want to repair all these streams (Y/N):")) {
        return -1;
    }
    for (String stream : streams) {
        fixInprogressSegmentWithLowerSequenceNumber(getNamespace(),
                metadataUpdater, stream, verbose, !getForce());
    }
    return 0;
}
 
Example 2
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    generateStreams(streamPrefix, streamExpression);
    if (streams.isEmpty()) {
        System.out.println("Nothing to create.");
        return 0;
    }
    if (!getForce() && !IOUtils.confirmPrompt("You are going to create streams : " + streams)) {
        return 0;
    }
    getConf().setZkAclId(getZkAclId());
    for (String stream : streams) {
        getFactory().getNamespace().createLog(stream);
    }
    return 0;
}
 
Example 3
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    MetadataUpdater metadataUpdater = dryrun ?
            new DryrunLogSegmentMetadataStoreUpdater(getConf(),
                    getLogSegmentMetadataStore()) :
            LogSegmentMetadataStoreUpdater.createMetadataUpdater(getConf(),
                    getLogSegmentMetadataStore());
    System.out.println("List of streams : ");
    System.out.println(streams);
    if (!IOUtils.confirmPrompt("Do you want to repair all these streams (Y/N):")) {
        return -1;
    }
    for (String stream : streams) {
        fixInprogressSegmentWithLowerSequenceNumber(getFactory(), metadataUpdater, stream, verbose, !getForce());
    }
    return 0;
}
 
Example 4
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to audit uris : "
            + getUris() + ", allocation paths = " + allocationPaths)) {
        return 0;
    }

    DLAuditor dlAuditor = new DLAuditor(getConf());
    try {
        Pair<Set<Long>, Set<Long>> bkdlLedgers = dlAuditor.collectLedgers(getUris(), allocationPaths);
        dumpLedgers(bkdlLedgers.getLeft(), new File(ledgersFilePrefix + "-bkledgers.txt"));
        dumpLedgers(bkdlLedgers.getRight(), new File(ledgersFilePrefix + "-dlledgers.txt"));
        dumpLedgers(Sets.difference(bkdlLedgers.getLeft(), bkdlLedgers.getRight()),
                    new File(ledgersFilePrefix + "-leakledgers.txt"));
    } finally {
        dlAuditor.close();
    }
    return 0;
}
 
Example 5
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 6 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to audit uris : "
            + getUris() + ", allocation paths = " + allocationPaths)) {
        return 0;
    }

    DLAuditor dlAuditor = new DLAuditor(getConf());
    try {
        Pair<Set<Long>, Set<Long>> bkdlLedgers = dlAuditor.collectLedgers(getUris(), allocationPaths);
        dumpLedgers(bkdlLedgers.getLeft(), new File(ledgersFilePrefix + "-bkledgers.txt"));
        dumpLedgers(bkdlLedgers.getRight(), new File(ledgersFilePrefix + "-dlledgers.txt"));
        dumpLedgers(Sets.difference(bkdlLedgers.getLeft(), bkdlLedgers.getRight()),
                    new File(ledgersFilePrefix + "-leakledgers.txt"));
    } finally {
        dlAuditor.close();
    }
    return 0;
}
 
Example 6
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
private int truncateStream(final Namespace namespace, String streamName, DLSN dlsn) throws Exception {
    DistributedLogManager dlm = namespace.openLog(streamName);
    try {
        long totalRecords = dlm.getLogRecordCount();
        long recordsAfterTruncate = FutureUtils.result(dlm.getLogRecordCountAsync(dlsn));
        long recordsToTruncate = totalRecords - recordsAfterTruncate;
        if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate "
                + streamName + " at dlsn " + dlsn + " (" + recordsToTruncate + " records)?")) {
            return 0;
        } else {
            AsyncLogWriter writer = dlm.startAsyncLogSegmentNonPartitioned();
            try {
                if (!FutureUtils.result(writer.truncate(dlsn))) {
                    System.out.println("Failed to truncate.");
                }
                return 0;
            } finally {
                Utils.close(writer);
            }
        }
    } catch (Exception ex) {
        System.err.println("Failed to truncate " + ex);
        return 1;
    } finally {
        dlm.close();
    }
}
 
Example 7
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
protected int inspectAndRepair(List<LogSegmentMetadata> segments) throws Exception {
    LogSegmentMetadataStore metadataStore = getLogSegmentMetadataStore();
    ZooKeeperClient zkc = getZooKeeperClient();
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, getUri());
    BKDLConfig.propagateConfiguration(bkdlConfig, getConf());
    BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder()
            .dlConfig(getConf())
            .zkServers(bkdlConfig.getBkZkServersForReader())
            .ledgersPath(bkdlConfig.getBkLedgersPath())
            .name("dlog")
            .build();
    try {
        List<LogSegmentMetadata> segmentsToRepair = inspectLogSegments(bkc, segments);
        if (segmentsToRepair.isEmpty()) {
            System.out.println("The stream is good. No log segments to repair.");
            return 0;
        }
        System.out.println(segmentsToRepair.size() + " segments to repair : ");
        System.out.println(segmentsToRepair);
        System.out.println();
        if (!IOUtils.confirmPrompt("Do you want to repair them (Y/N): ")) {
            return 0;
        }
        repairLogSegments(metadataStore, bkc, segmentsToRepair);
        return 0;
    } finally {
        bkc.close();
    }
}
 
Example 8
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
protected int runCmd(CommandLine cmdline) throws Exception {
    String[] args = cmdline.getArgs();
    if (args.length <= 0) {
        System.err.println("No distributedlog uri specified.");
        printUsage();
        return -1;
    }
    boolean force = cmdline.hasOption("f");
    URI uri = URI.create(args[0]);
    // resolving the uri to see if there is another bindings in this uri.
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri)
            .sessionTimeoutMs(10000).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException ie) {
        bkdlConfig = null;
    }
    if (null == bkdlConfig) {
        System.out.println("No bookkeeper is bound to " + uri);
        return 0;
    } else {
        System.out.println("There is bookkeeper bound to " + uri + " : ");
        System.out.println("");
        System.out.println(bkdlConfig.toString());
        System.out.println("");
        if (!force && !IOUtils.confirmPrompt("Do you want to unbind " + uri + " :\n")) {
            return 0;
        }
    }
    DLMetadata.unbind(uri);
    System.out.println("Unbound on " + uri + ".");
    return 0;
}
 
Example 9
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
protected void repairLogSegment(BookKeeperAdmin bkAdmin,
                                MetadataUpdater metadataUpdater,
                                LogSegmentMetadata segment) throws Exception {
    if (segment.isInProgress()) {
        System.out.println("Skip inprogress log segment " + segment);
        return;
    }
    LedgerHandle lh = bkAdmin.openLedger(segment.getLogSegmentId());
    long lac = lh.getLastAddConfirmed();
    Enumeration<LedgerEntry> entries = lh.readEntries(lac, lac);
    if (!entries.hasMoreElements()) {
        throw new IOException("Entry " + lac + " isn't found for " + segment);
    }
    LedgerEntry lastEntry = entries.nextElement();
    Entry.Reader reader = Entry.newBuilder()
            .setLogSegmentInfo(segment.getLogSegmentSequenceNumber(), segment.getStartSequenceId())
            .setEntryId(lastEntry.getEntryId())
            .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(segment.getVersion()))
            .setEntry(lastEntry.getEntryBuffer())
            .buildReader();
    lastEntry.getEntryBuffer().release();
    LogRecordWithDLSN record = reader.nextRecord();
    LogRecordWithDLSN lastRecord = null;
    while (null != record) {
        lastRecord = record;
        record = reader.nextRecord();
    }
    if (null == lastRecord) {
        throw new IOException("No record found in entry " + lac + " for " + segment);
    }
    System.out.println("Updating last record for " + segment + " to " + lastRecord);
    if (!IOUtils.confirmPrompt("Do you want to make this change (Y/N): ")) {
        return;
    }
    metadataUpdater.updateLastRecord(segment, lastRecord);
}
 
Example 10
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
protected int inspectAndRepair(List<LogSegmentMetadata> segments) throws Exception {
    LogSegmentMetadataStore metadataStore = getLogSegmentMetadataStore();
    ZooKeeperClient zkc = getZooKeeperClient();
    BKDLConfig bkdlConfig = BKDLConfig.resolveDLConfig(zkc, getUri());
    BKDLConfig.propagateConfiguration(bkdlConfig, getConf());
    BookKeeperClient bkc = BookKeeperClientBuilder.newBuilder()
            .dlConfig(getConf())
            .zkServers(bkdlConfig.getBkZkServersForReader())
            .ledgersPath(bkdlConfig.getBkLedgersPath())
            .name("dlog")
            .build();
    try {
        List<LogSegmentMetadata> segmentsToRepair = inspectLogSegments(bkc, segments);
        if (segmentsToRepair.isEmpty()) {
            System.out.println("The stream is good. No log segments to repair.");
            return 0;
        }
        System.out.println(segmentsToRepair.size() + " segments to repair : ");
        System.out.println(segmentsToRepair);
        System.out.println();
        if (!IOUtils.confirmPrompt("Do you want to repair them (Y/N): ")) {
            return 0;
        }
        repairLogSegments(metadataStore, bkc, segmentsToRepair);
        return 0;
    } finally {
        bkc.close();
    }
}
 
Example 11
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 5 votes vote down vote up
@Override
protected int runCmd(CommandLine cmdline) throws Exception {
    String[] args = cmdline.getArgs();
    if (args.length <= 0) {
        System.err.println("No distributedlog uri specified.");
        printUsage();
        return -1;
    }
    boolean force = cmdline.hasOption("f");
    URI uri = URI.create(args[0]);
    // resolving the uri to see if there is another bindings in this uri.
    ZooKeeperClient zkc = ZooKeeperClientBuilder.newBuilder().uri(uri)
            .sessionTimeoutMs(10000).build();
    BKDLConfig bkdlConfig;
    try {
        bkdlConfig = BKDLConfig.resolveDLConfig(zkc, uri);
    } catch (IOException ie) {
        bkdlConfig = null;
    }
    if (null == bkdlConfig) {
        System.out.println("No bookkeeper is bound to " + uri);
        return 0;
    } else {
        System.out.println("There is bookkeeper bound to " + uri + " : ");
        System.out.println("");
        System.out.println(bkdlConfig.toString());
        System.out.println("");
        if (!force && !IOUtils.confirmPrompt("Do you want to unbind " + uri + " :\n")) {
            return 0;
        }
    }
    DLMetadata.unbind(uri);
    System.out.println("Unbound on " + uri + ".");
    return 0;
}
 
Example 12
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
public static void checkAndRepairDLNamespace(final URI uri,
                                             final Namespace namespace,
                                             final MetadataUpdater metadataUpdater,
                                             final OrderedScheduler scheduler,
                                             final boolean verbose,
                                             final boolean interactive,
                                             final int concurrency) throws Exception {
    checkArgument(concurrency > 0, "Invalid concurrency " + concurrency + " found.");
    // 0. getting streams under a given uri.
    Iterator<String> streamsIter = namespace.getLogs();
    List<String> streams = Lists.newArrayList();
    while (streamsIter.hasNext()) {
        streams.add(streamsIter.next());
    }
    if (verbose) {
        System.out.println("- 0. checking streams under " + uri);
    }
    if (streams.size() == 0) {
        System.out.println("+ 0. nothing to check. quit.");
        return;
    }
    Map<String, StreamCandidate> streamCandidates =
            checkStreams(namespace, streams, scheduler, concurrency);
    if (verbose) {
        System.out.println("+ 0. " + streamCandidates.size() + " corrupted streams found.");
    }
    if (interactive && !IOUtils.confirmPrompt("Do you want to fix all "
            + streamCandidates.size() + " corrupted streams (Y/N) : ")) {
        return;
    }
    if (verbose) {
        System.out.println("- 1. repairing " + streamCandidates.size() + " corrupted streams.");
    }
    for (StreamCandidate candidate : streamCandidates.values()) {
        if (!repairStream(metadataUpdater, candidate, verbose, interactive)) {
            if (verbose) {
                System.out.println("* 1. aborted repairing corrupted streams.");
            }
            return;
        }
    }
    if (verbose) {
        System.out.println("+ 1. repaired " + streamCandidates.size() + " corrupted streams.");
    }
}
 
Example 13
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
private int deleteSubscriber(final Namespace namespace) throws Exception {
    Iterator<String> streamCollection = namespace.getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return 0;
    }
    System.out.println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to delete subscriber "
        + subscriberId + " for " + streams.size() + " streams ?")) {
        return 0;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads + 1;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("RemoveSubscriberThread-" + i) {
            @Override
            public void run() {
                try {
                    deleteSubscriber(namespace, streams, tid, numStreamsPerThreads);
                    System.out.println("Thread " + tid + " finished.");
                } catch (Exception e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    return 0;
}
 
Example 14
Source File: DistributedLogAdmin.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
public static void checkAndRepairDLNamespace(final URI uri,
                                             final com.twitter.distributedlog.DistributedLogManagerFactory factory,
                                             final MetadataUpdater metadataUpdater,
                                             final ExecutorService executorService,
                                             final BookKeeperClient bkc,
                                             final String digestpw,
                                             final boolean verbose,
                                             final boolean interactive,
                                             final int concurrency) throws IOException {
    Preconditions.checkArgument(concurrency > 0, "Invalid concurrency " + concurrency + " found.");
    // 0. getting streams under a given uri.
    Collection<String> streams = factory.enumerateAllLogsInNamespace();
    if (verbose) {
        System.out.println("- 0. checking " + streams.size() + " streams under " + uri);
    }
    if (streams.size() == 0) {
        System.out.println("+ 0. nothing to check. quit.");
        return;
    }
    Map<String, StreamCandidate> streamCandidates =
            checkStreams(factory, streams, executorService, bkc, digestpw, concurrency);
    if (verbose) {
        System.out.println("+ 0. " + streamCandidates.size() + " corrupted streams found.");
    }
    if (interactive && !IOUtils.confirmPrompt("Do you want to fix all " + streamCandidates.size() + " corrupted streams (Y/N) : ")) {
        return;
    }
    if (verbose) {
        System.out.println("- 1. repairing " + streamCandidates.size() + " corrupted streams.");
    }
    for (StreamCandidate candidate : streamCandidates.values()) {
        if (!repairStream(metadataUpdater, candidate, verbose, interactive)) {
            if (verbose) {
                System.out.println("* 1. aborted repairing corrupted streams.");
            }
            return;
        }
    }
    if (verbose) {
        System.out.println("+ 1. repaired " + streamCandidates.size() + " corrupted streams.");
    }
}
 
Example 15
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
private int truncateStreams(final Namespace namespace) throws Exception {
    Iterator<String> streamCollection = namespace.getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return 0;
    }
    System.out.println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate " + streams.size() + " streams ?")) {
        return 0;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads + 1;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("Truncate-" + i) {
            @Override
            public void run() {
                try {
                    truncateStreams(namespace, streams, tid, numStreamsPerThreads);
                    System.out.println("Thread " + tid + " finished.");
                } catch (IOException e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    return 0;
}
 
Example 16
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
private void inspectStreams(final SortedMap<String,
        List<Pair<LogSegmentMetadata, List<String>>>> corruptedCandidates) throws Exception {
    Iterator<String> streamCollection = getNamespace().getLogs();
    final List<String> streams = new ArrayList<String>();
    while (streamCollection.hasNext()) {
        String s = streamCollection.next();
        if (null != streamPrefix) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        } else {
            streams.add(s);
        }
    }
    if (0 == streams.size()) {
        return;
    }
    println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Are you sure you want to inspect "
            + streams.size() + " streams")) {
        return;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("Inspect-" + i) {
            @Override
            public void run() {
                try {
                    inspectStreams(streams, tid, numStreamsPerThreads, corruptedCandidates);
                    System.out.println("Thread " + tid + " finished.");
                } catch (Exception e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
}
 
Example 17
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    String rootPath = getUri().getPath() + "/" + allocationPoolPath;
    final ScheduledExecutorService allocationExecutor = Executors.newSingleThreadScheduledExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    try {
        List<String> pools = getZooKeeperClient().get().getChildren(rootPath, false);
        final LinkedBlockingQueue<String> poolsToDelete = new LinkedBlockingQueue<String>();
        if (getForce() || IOUtils.confirmPrompt("Are you sure you want to delete allocator pools : " + pools)) {
            for (String pool : pools) {
                poolsToDelete.add(rootPath + "/" + pool);
            }
            final CountDownLatch doneLatch = new CountDownLatch(concurrency);
            for (int i = 0; i < concurrency; i++) {
                final int tid = i;
                executorService.submit(new Runnable() {
                    @Override
                    public void run() {
                        while (!poolsToDelete.isEmpty()) {
                            String poolPath = poolsToDelete.poll();
                            if (null == poolPath) {
                                break;
                            }
                            try {
                                LedgerAllocator allocator =
                                        LedgerAllocatorUtils.createLedgerAllocatorPool(poolPath, 0, getConf(),
                                                getZooKeeperClient(), getBookKeeperClient(),
                                                allocationExecutor);
                                allocator.delete();
                                System.out.println("Deleted allocator pool : " + poolPath + " .");
                            } catch (IOException ioe) {
                                System.err.println("Failed to delete allocator pool " + poolPath + " : " + ioe.getMessage());
                            }
                        }
                        doneLatch.countDown();
                        System.out.println("Thread " + tid + " is done.");
                    }
                });
            }
            doneLatch.await();
        }
    } finally {
        executorService.shutdown();
        allocationExecutor.shutdown();
    }
    return 0;
}
 
Example 18
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
@Override
protected int runBKCmd(ZooKeeperClient zkc, BookKeeperClient bkc) throws Exception {
    BookKeeperAdmin bkAdmin = new BookKeeperAdmin(bkc.get());
    try {
        if (query) {
            return bkQuery(bkAdmin, bookiesSrc);
        }
        if (fenceOnly) {
            return bkFence(bkc, ledgers, fenceRate);
        }
        if (!force) {
            System.out.println("Bookies : " + bookiesSrc);
            if (!IOUtils.confirmPrompt("Do you want to recover them: (Y/N)")) {
                return -1;
            }
        }
        if (!ledgers.isEmpty()) {
            System.out.println("Ledgers : " + ledgers);
            long numProcessed = 0;
            Iterator<Long> ledgersIter = ledgers.iterator();
            LinkedBlockingQueue<Long> ledgersToProcess = new LinkedBlockingQueue<Long>();
            while (ledgersIter.hasNext()) {
                long lid = ledgersIter.next();
                if (numPartitions <=0 || (numPartitions > 0 && lid % numPartitions == partition)) {
                    ledgersToProcess.add(lid);
                    ++numProcessed;
                }
                if (ledgersToProcess.size() == 10000) {
                    System.out.println("Processing " + numProcessed + " ledgers");
                    bkRecovery(ledgersToProcess, bookiesSrc, dryrun, skipOpenLedgers);
                    ledgersToProcess.clear();
                    System.out.println("Processed " + numProcessed + " ledgers");
                }
            }
            if (!ledgersToProcess.isEmpty()) {
                System.out.println("Processing " + numProcessed + " ledgers");
                bkRecovery(ledgersToProcess, bookiesSrc, dryrun, skipOpenLedgers);
                System.out.println("Processed " + numProcessed + " ledgers");
            }
            System.out.println("Done.");
            CountDownLatch latch = new CountDownLatch(1);
            latch.await();
            return 0;
        }
        return bkRecovery(bkAdmin, bookiesSrc, dryrun, skipOpenLedgers);
    } finally {
        bkAdmin.close();
    }
}
 
Example 19
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
private int truncateStreams(final com.twitter.distributedlog.DistributedLogManagerFactory factory) throws Exception {
    Collection<String> streamCollection = factory.enumerateAllLogsInNamespace();
    final List<String> streams = new ArrayList<String>();
    if (null != streamPrefix) {
        for (String s : streamCollection) {
            if (s.startsWith(streamPrefix)) {
                streams.add(s);
            }
        }
    } else {
        streams.addAll(streamCollection);
    }
    if (0 == streams.size()) {
        return 0;
    }
    System.out.println("Streams : " + streams);
    if (!getForce() && !IOUtils.confirmPrompt("Do you want to truncate " + streams.size() + " streams ?")) {
        return 0;
    }
    numThreads = Math.min(streams.size(), numThreads);
    final int numStreamsPerThreads = streams.size() / numThreads;
    Thread[] threads = new Thread[numThreads];
    for (int i = 0; i < numThreads; i++) {
        final int tid = i;
        threads[i] = new Thread("Truncate-" + i) {
            @Override
            public void run() {
                try {
                    truncateStreams(factory, streams, tid, numStreamsPerThreads);
                    System.out.println("Thread " + tid + " finished.");
                } catch (IOException e) {
                    System.err.println("Thread " + tid + " quits with exception : " + e.getMessage());
                }
            }
        };
        threads[i].start();
    }
    for (int i = 0; i < numThreads; i++) {
        threads[i].join();
    }
    return 0;
}
 
Example 20
Source File: DistributedLogTool.java    From distributedlog with Apache License 2.0 4 votes vote down vote up
@Override
protected int runCmd() throws Exception {
    String rootPath = getUri().getPath() + "/" + allocationPoolPath;
    final ScheduledExecutorService allocationExecutor = Executors.newSingleThreadScheduledExecutor();
    ExecutorService executorService = Executors.newFixedThreadPool(concurrency);
    checkArgument(getNamespace() instanceof BKDistributedLogNamespace);
    BKDistributedLogNamespace bkns = (BKDistributedLogNamespace) getNamespace();
    final ZooKeeperClient zkc = ((BKNamespaceDriver) bkns.getNamespaceDriver()).getWriterZKC();
    final BookKeeperClient bkc = ((BKNamespaceDriver) bkns.getNamespaceDriver()).getReaderBKC();
    try {
        List<String> pools = zkc.get().getChildren(rootPath, false);
        final LinkedBlockingQueue<String> poolsToDelete = new LinkedBlockingQueue<String>();
        if (getForce() || IOUtils.confirmPrompt("Are you sure you want to delete allocator pools : " + pools)) {
            for (String pool : pools) {
                poolsToDelete.add(rootPath + "/" + pool);
            }
            final CountDownLatch doneLatch = new CountDownLatch(concurrency);
            for (int i = 0; i < concurrency; i++) {
                final int tid = i;
                executorService.submit(new Runnable() {
                    @Override
                    public void run() {
                        while (!poolsToDelete.isEmpty()) {
                            String poolPath = poolsToDelete.poll();
                            if (null == poolPath) {
                                break;
                            }
                            try {
                                LedgerAllocator allocator =
                                        LedgerAllocatorUtils.createLedgerAllocatorPool(poolPath, 0, getConf(),
                                                zkc, bkc,
                                                allocationExecutor);
                                allocator.delete();
                                System.out.println("Deleted allocator pool : " + poolPath + " .");
                            } catch (IOException ioe) {
                                System.err.println("Failed to delete allocator pool "
                                        + poolPath + " : " + ioe.getMessage());
                            }
                        }
                        doneLatch.countDown();
                        System.out.println("Thread " + tid + " is done.");
                    }
                });
            }
            doneLatch.await();
        }
    } finally {
        executorService.shutdown();
        allocationExecutor.shutdown();
    }
    return 0;
}