Java Code Examples for com.twitter.util.Future#collect()

The following examples show how to use com.twitter.util.Future#collect() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StreamManagerImpl.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private Future<List<Void>> closeStreams(Set<Stream> streamsToClose, Optional<RateLimiter> rateLimiter) {
    if (streamsToClose.isEmpty()) {
        logger.info("No streams to close.");
        List<Void> emptyList = new ArrayList<Void>();
        return Future.value(emptyList);
    }
    List<Future<Void>> futures = new ArrayList<Future<Void>>(streamsToClose.size());
    for (Stream stream : streamsToClose) {
        if (rateLimiter.isPresent()) {
            rateLimiter.get().acquire();
        }
        futures.add(stream.requestClose("Close Streams"));
    }
    return Future.collect(futures);
}
Example 2
Source File: ZKLogMetadataForWriter.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
static Future<List<Versioned<byte[]>>> checkLogMetadataPaths(ZooKeeper zk,
                                                             String logRootPath,
                                                             boolean ownAllocator) {
    // Note re. persistent lock state initialization: the read lock persistent state (path) is
    // initialized here but only used in the read handler. The reason is its more convenient and
    // less error prone to manage all stream structure in one place.
    final String logRootParentPath = new File(logRootPath).getParent();
    final String logSegmentsPath = logRootPath + LOGSEGMENTS_PATH;
    final String maxTxIdPath = logRootPath + MAX_TXID_PATH;
    final String lockPath = logRootPath + LOCK_PATH;
    final String readLockPath = logRootPath + READ_LOCK_PATH;
    final String versionPath = logRootPath + VERSION_PATH;
    final String allocationPath = logRootPath + ALLOCATION_PATH;

    int numPaths = ownAllocator ? MetadataIndex.ALLOCATION + 1 : MetadataIndex.LOGSEGMENTS + 1;
    List<Future<Versioned<byte[]>>> checkFutures = Lists.newArrayListWithExpectedSize(numPaths);
    checkFutures.add(Utils.zkGetData(zk, logRootParentPath, false));
    checkFutures.add(Utils.zkGetData(zk, logRootPath, false));
    checkFutures.add(Utils.zkGetData(zk, maxTxIdPath, false));
    checkFutures.add(Utils.zkGetData(zk, versionPath, false));
    checkFutures.add(Utils.zkGetData(zk, lockPath, false));
    checkFutures.add(Utils.zkGetData(zk, readLockPath, false));
    checkFutures.add(Utils.zkGetData(zk, logSegmentsPath, false));
    if (ownAllocator) {
        checkFutures.add(Utils.zkGetData(zk, allocationPath, false));
    }

    return Future.collect(checkFutures);
}
Example 3
Source File: FederatedZKLogMetadataStore.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private Future<List<Set<String>>> retrieveLogs() {
    Collection<SubNamespace> subNss = subNamespaces.values();
    List<Future<Set<String>>> logsList = Lists.newArrayListWithExpectedSize(subNss.size());
    for (SubNamespace subNs : subNss) {
        logsList.add(subNs.getLogs());
    }
    return Future.collect(logsList);
}
Example 4
Source File: StreamManagerImpl.java    From distributedlog with Apache License 2.0 5 votes vote down vote up 
private Future<List<Void>> closeStreams(Set<Stream> streamsToClose, Optional<RateLimiter> rateLimiter) {
    if (streamsToClose.isEmpty()) {
        logger.info("No streams to close.");
        List<Void> emptyList = new ArrayList<Void>();
        return Future.value(emptyList);
    }
    List<Future<Void>> futures = new ArrayList<Future<Void>>(streamsToClose.size());
    for (Stream stream : streamsToClose) {
        if (rateLimiter.isPresent()) {
            rateLimiter.get().acquire();
        }
        futures.add(stream.requestClose("Close Streams"));
    }
    return Future.collect(futures);
}
Example 5
Source File: CoreLindenCluster.java    From linden with Apache License 2.0 4 votes vote down vote up 
@Override
public Response delete(LindenDeleteRequest request) throws IOException {
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  Set<Integer> routeIds = null;
  if (request.isSetRouteParam()) {
    routeIds = new HashSet<>();
    for (final ShardRouteParam routeParam : request.getRouteParam().getShardParams()) {
      routeIds.add(routeParam.getShardId());
    }
  }
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    if (routeIds != null && !routeIds.contains(entry.getKey())) {
      continue;
    }
    if (entry.getValue().isAvailable()) {
      List<Map.Entry<String, Future<Response>>> hostFuturePairs = entry.getValue().delete(request);
      for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
        hosts.add(hostFuturePair.getKey());
        futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
          @Override
          public BoxedUnit map(Response response) {
            if (!response.isSuccess()) {
              synchronized (errorInfo) {
                LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}",
                             entry.getKey(), hostFuturePair.getKey(), response.getError());
                errorInfo
                    .append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError()
                            + ";");

              }
            }
            return BoxedUnit.UNIT;
          }

          @Override
          public BoxedUnit handle(Throwable t) {
            LOGGER.error("Shard [{}] host [{}] failed to get delete response : {}",
                         entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
            errorInfo
                .append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                    .getStackTraceAsString(t) + ";");
            return BoxedUnit.UNIT;
          }
        }));
      }
    }
  }

  Future<List<BoxedUnit>> collected = Future.collect(futures);
  try {
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("Delete failed: " + errorInfo.toString());
    }
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Failed to get all delete responses, exception: {}", Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
Example 6
Source File: CoreLindenCluster.java    From linden with Apache License 2.0 4 votes vote down vote up 
@Override
public Response index(String content) throws IOException {
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  LindenIndexRequest indexRequest = LindenIndexRequestParser.parse(lindenConfig.getSchema(), content);
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    ShardClient shardClient = entry.getValue();
    if (shardClient.isAvailable()) {
      if (shardingStrategy
          .accept(indexRequest.getId(), indexRequest.getRouteParam(), shardClient.getShardId())) {
        final List<Map.Entry<String, Future<Response>>> hostFuturePairs = shardClient.index(content);
        for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
          hosts.add(hostFuturePair.getKey());
          futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
            @Override
            public BoxedUnit map(Response response) {
              if (!response.isSuccess()) {
                LOGGER.error("Shard [{}] host [{}] failed to get index response : {}",
                             entry.getKey(), hostFuturePair.getKey(), response.getError());
                synchronized (errorInfo) {
                  errorInfo.append(
                      "Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError()
                      + ";");
                }
              }
              return BoxedUnit.UNIT;
            }

            @Override
            public BoxedUnit handle(Throwable t) {
              LOGGER.error("Shard [{}] host [{}] failed to get index response : {}",
                           entry.getKey(), hostFuturePair.getKey(), Throwables.getStackTraceAsString(t));
              synchronized (errorInfo) {
                errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                    .getStackTraceAsString(t) + ";");
              }
              return BoxedUnit.UNIT;
            }
          }));
        }
      }
    }
  }
  try {
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      Await.result(collected, Duration.apply(clusterFutureAwaitTimeout, TimeUnit.MILLISECONDS));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("Index failed: " + errorInfo.toString());
    }
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Handle request failed, content : {} - {}", content, Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
Example 7
Source File: CoreLindenCluster.java    From linden with Apache License 2.0 4 votes vote down vote up 
@Override
public Response executeCommand(final String command) throws IOException {
  LOGGER.info("Receive cluster command {}", command);
  List<Future<BoxedUnit>> futures = new ArrayList<>();
  List<String> hosts = new ArrayList<>();
  final StringBuilder errorInfo = new StringBuilder();
  JSONObject jsonCmd = JSONObject.parseObject(command);
  Integer shardId = jsonCmd.getInteger(SHARD);
  for (final Map.Entry<Integer, ShardClient> entry : clients.entrySet()) {
    ShardClient shardClient = entry.getValue();
    if (shardId != null && shardId != shardClient.getShardId()) {
      continue;
    }
    if (shardClient.isAvailable()) {
      final List<Map.Entry<String, Future<Response>>> hostFuturePairs = shardClient.executeCommand(jsonCmd);
      for (final Map.Entry<String, Future<Response>> hostFuturePair : hostFuturePairs) {
        hosts.add(hostFuturePair.getKey());
        futures.add(hostFuturePair.getValue().transformedBy(new FutureTransformer<Response, BoxedUnit>() {
          @Override
          public BoxedUnit map(Response response) {
            if (!response.isSuccess()) {
              LOGGER.error("Shard [{}] host [{}] failed to execute command {} error: {}",
                           entry.getKey(), hostFuturePair.getKey(), command, response.getError());
              synchronized (errorInfo) {
                errorInfo.append(
                    "Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + response.getError() + ";");
              }
            }
            return BoxedUnit.UNIT;
          }

          @Override
          public BoxedUnit handle(Throwable t) {
            LOGGER.error("Shard [{}] host [{}] failed to execute command {} throwable: {}\"",
                         entry.getKey(), hostFuturePair.getKey(), command, Throwables.getStackTraceAsString(t));
            synchronized (errorInfo) {
              errorInfo.append("Shard " + entry.getKey() + " host " + hostFuturePair.getKey() + ":" + Throwables
                  .getStackTraceAsString(t) + ";");
            }
            return BoxedUnit.UNIT;
          }
        }));
      }
    }
  }
  try {
    Future<List<BoxedUnit>> collected = Future.collect(futures);
    if (clusterFutureAwaitTimeout == 0) {
      Await.result(collected);
    } else {
      // executeCommand may take a very long time, so set timeout to 30min specially
      Await.result(collected, Duration.apply(30, TimeUnit.MINUTES));
    }
    if (errorInfo.length() > 0) {
      return ResponseUtils.buildFailedResponse("command " + command + " failed: " + errorInfo.toString());
    }
    LOGGER.error("Cluster command {} succeeded", command);
    return ResponseUtils.SUCCESS;
  } catch (Exception e) {
    LOGGER.error("Cluster command {} failed {}", command, Throwables.getStackTraceAsString(e));
    LOGGER.error(getHostFutureInfo(hosts, futures));
    return ResponseUtils.buildFailedResponse(e);
  }
}
Example 8
Source File: MultiLindenCoreImpl.java    From linden with Apache License 2.0 4 votes vote down vote up 
@Override
public LindenResult search(final LindenSearchRequest request) throws IOException {
  final List<LindenResult> resultList = new ArrayList<>();
  final List<Future<BoxedUnit>> futures = new ArrayList<>();

  List<String> indexNames;
  // Only INDEX_NAME division type supports specified index name request
  if (multiIndexStrategy instanceof TimeLimitMultiIndexStrategy
      || multiIndexStrategy instanceof DocNumLimitMultiIndexStrategy) {
    indexNames = new ArrayList<>(lindenCoreMap.keySet());
  } else {
    if (request.getIndexNames() == null || (request.getIndexNamesSize() == 1 && request.getIndexNames().get(0)
        .equals(LINDEN))) {
      indexNames = new ArrayList<>(lindenCoreMap.keySet());
    } else {
      indexNames = request.getIndexNames();
      for (int i = 0; i < indexNames.size(); ++i) {
        indexNames.set(i, MultiIndexStrategy.MULTI_INDEX_PREFIX_NAME + indexNames.get(i));
      }
    }
  }

  for (final String indexName : indexNames) {
    final LindenCore core = lindenCoreMap.get(indexName);
    if (core != null) {
      futures
          .add(Future.value(core.search(request)).transformedBy(new FutureTransformer<LindenResult, BoxedUnit>() {
            @Override
            public BoxedUnit map(LindenResult lindenResult) {
              synchronized (resultList) {
                resultList.add(lindenResult);
              }
              return BoxedUnit.UNIT;
            }

            @Override
            public BoxedUnit handle(Throwable t) {
              LOGGER.info("Index {} search error: {}", indexName,
                          Throwables.getStackTraceAsString(t));
              return BoxedUnit.UNIT;
            }
          }));
    } else {
      LOGGER.error("Index {} doesn't exist.", indexName);
    }
  }
  Future<List<BoxedUnit>> collected = Future.collect(futures);
  try {
    collected.apply(Duration.apply(DEFAULT_SEARCH_TIMEOUT, TimeUnit.MILLISECONDS));
  } catch (Exception e) {
    LOGGER.error("Multi-index search error: {}", Throwables.getStackTraceAsString(e));
  }
  return ResultMerger.merge(request, resultList);
}