org.apache.lucene.util.CollectionUtil Java Examples

The following examples show how to use org.apache.lucene.util.CollectionUtil. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TransportCreatePartitionsAction.java    From crate with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(CreatePartitionsRequest request, ClusterState state) {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    String firstIndex = request.indices().iterator().next();


    // note: only use the first index name to see if template matches.
    // this means
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;
        for (String pattern : template.getPatterns()) {
            if (Regex.simpleMatch(pattern, firstIndex)) {
                templates.add(template);
                break;
            }
        }
    }
    CollectionUtil.timSort(templates, (o1, o2) -> o2.order() - o1.order());
    return templates;
}
 
Example #2
Source File: InternalHistogram.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public InternalAggregation doReduce(List<InternalAggregation> aggregations, ReduceContext reduceContext) {
    List<B> reducedBuckets = reduceBuckets(aggregations, reduceContext);

    // adding empty buckets if needed
    if (minDocCount == 0) {
        addEmptyBuckets(reducedBuckets, reduceContext);
    }

    if (order == InternalOrder.KEY_ASC) {
        // nothing to do, data are already sorted since shards return
        // sorted buckets and the merge-sort performed by reduceBuckets
        // maintains order
    } else if (order == InternalOrder.KEY_DESC) {
        // we just need to reverse here...
        List<B> reverse = new ArrayList<>(reducedBuckets);
        Collections.reverse(reverse);
        reducedBuckets = reverse;
    } else {
        // sorted by sub-aggregation, need to fall back to a costly n*log(n) sort
        CollectionUtil.introSort(reducedBuckets, order.comparator());
    }

    return getFactory().create(getName(), reducedBuckets, order, minDocCount, emptyBucketInfo, formatter, keyed, pipelineAggregators(),
            getMetaData());
}
 
Example #3
Source File: SnapshotsService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @return list of snapshots
 */
public List<Snapshot> snapshots(String repositoryName, boolean ignoreUnavailable) {
    Set<Snapshot> snapshotSet = newHashSet();
    List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
    }
    Repository repository = repositoriesService.repository(repositoryName);
    List<SnapshotId> snapshotIds = repository.snapshots();
    for (SnapshotId snapshotId : snapshotIds) {
        try {
            snapshotSet.add(repository.readSnapshot(snapshotId));
        } catch (Exception ex) {
            if (ignoreUnavailable) {
                logger.warn("failed to get snapshot [{}]", ex, snapshotId);
            } else {
                throw new SnapshotException(snapshotId, "Snapshot could not be read", ex);
            }
        }
    }

    ArrayList<Snapshot> snapshotList = new ArrayList<>(snapshotSet);
    CollectionUtil.timSort(snapshotList);
    return Collections.unmodifiableList(snapshotList);
}
 
Example #4
Source File: MetaDataCreateIndexService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;
        if (indexTemplateFilter.apply(request, template)) {
            templates.add(template);
        }
    }

    CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
        @Override
        public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {
            return o2.order() - o1.order();
        }
    });
    return templates;
}
 
Example #5
Source File: TransportBulkCreateIndicesAction.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(BulkCreateIndicesRequest request,
                                                  ClusterState state,
                                                  IndexTemplateFilter indexTemplateFilter) {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    CreateIndexClusterStateUpdateRequest dummyRequest =
            new CreateIndexClusterStateUpdateRequest(request, "bulk-create", request.indices().iterator().next(), false);

    // note: only use the first index name to see if template matches.
    // this means
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;

        if (indexTemplateFilter.apply(dummyRequest, template)) {
            templates.add(template);
        }
    }
    CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
        @Override
        public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {
            return o2.order() - o1.order();
        }
    });
    return templates;
}
 
Example #6
Source File: Errors.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public List<Message> getMessages() {
    if (root.errors == null) {
        return Collections.emptyList();
    }

    List<Message> result = new ArrayList<>(root.errors);
    CollectionUtil.timSort(result, new Comparator<Message>() {
        @Override
        public int compare(Message a, Message b) {
            return a.getSource().compareTo(b.getSource());
        }
    });

    return result;
}
 
Example #7
Source File: SnapshotsService.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a list of currently running snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @return list of snapshots
 */
public List<SnapshotInfo> currentSnapshots(final String repositoryName) {
    List<SnapshotInfo> snapshotList = new ArrayList<>();
    List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, Collections.emptyList());
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotList.add(inProgressSnapshot(entry));
    }
    CollectionUtil.timSort(snapshotList);
    return unmodifiableList(snapshotList);
}
 
Example #8
Source File: SnapshotsService.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a list of snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @param snapshotIds       snapshots for which to fetch snapshot information
 * @param ignoreUnavailable if true, snapshots that could not be read will only be logged with a warning,
 *                          if false, they will throw an error
 * @return list of snapshots
 */
public List<SnapshotInfo> snapshots(final String repositoryName,
                                    final List<SnapshotId> snapshotIds,
                                    final boolean ignoreUnavailable) {
    final Set<SnapshotInfo> snapshotSet = new HashSet<>();
    final Set<SnapshotId> snapshotIdsToIterate = new HashSet<>(snapshotIds);
    // first, look at the snapshots in progress
    final List<SnapshotsInProgress.Entry> entries =
        currentSnapshots(repositoryName, snapshotIdsToIterate.stream().map(SnapshotId::getName).collect(Collectors.toList()));
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotSet.add(inProgressSnapshot(entry));
        snapshotIdsToIterate.remove(entry.snapshot().getSnapshotId());
    }
    // then, look in the repository
    final Repository repository = repositoriesService.repository(repositoryName);
    for (SnapshotId snapshotId : snapshotIdsToIterate) {
        try {
            snapshotSet.add(repository.getSnapshotInfo(snapshotId));
        } catch (Exception ex) {
            if (ignoreUnavailable) {
                LOGGER.warn(() -> new ParameterizedMessage("failed to get snapshot [{}]", snapshotId), ex);
            } else {
                if (ex instanceof SnapshotException) {
                    throw ex;
                }
                throw new SnapshotException(repositoryName, snapshotId, "Snapshot could not be read", ex);
            }
        }
    }
    final ArrayList<SnapshotInfo> snapshotList = new ArrayList<>(snapshotSet);
    CollectionUtil.timSort(snapshotList);
    return unmodifiableList(snapshotList);
}
 
Example #9
Source File: MetaDataIndexTemplateService.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Finds index templates whose index pattern matched with the given index name.
 * The result is sorted by {@link IndexTemplateMetaData#order} descending.
 */
public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName) {
    final List<IndexTemplateMetaData> matchedTemplates = new ArrayList<>();
    for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
        final IndexTemplateMetaData template = cursor.value;
        final boolean matched = template.patterns().stream().anyMatch(pattern -> Regex.simpleMatch(pattern, indexName));
        if (matched) {
            matchedTemplates.add(template);
        }
    }
    CollectionUtil.timSort(matchedTemplates, Comparator.comparingInt(IndexTemplateMetaData::order).reversed());
    return matchedTemplates;
}
 
Example #10
Source File: TransportIndicesShardStoresAction.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
void finish() {
    ImmutableOpenMap.Builder<String, ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>>> indicesStoreStatusesBuilder = ImmutableOpenMap.builder();
    java.util.List<IndicesShardStoresResponse.Failure> failureBuilder = new ArrayList<>();
    for (Response fetchResponse : fetchResponses) {
        ImmutableOpenIntMap<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexStoreStatuses = indicesStoreStatusesBuilder.get(fetchResponse.shardId.getIndex());
        final ImmutableOpenIntMap.Builder<java.util.List<IndicesShardStoresResponse.StoreStatus>> indexShardsBuilder;
        if (indexStoreStatuses == null) {
            indexShardsBuilder = ImmutableOpenIntMap.builder();
        } else {
            indexShardsBuilder = ImmutableOpenIntMap.builder(indexStoreStatuses);
        }
        java.util.List<IndicesShardStoresResponse.StoreStatus> storeStatuses = indexShardsBuilder.get(fetchResponse.shardId.id());
        if (storeStatuses == null) {
            storeStatuses = new ArrayList<>();
        }
        for (NodeGatewayStartedShards response : fetchResponse.responses) {
            if (shardExistsInNode(response)) {
                IndicesShardStoresResponse.StoreStatus.Allocation allocation = getAllocation(fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), response.getNode());
                storeStatuses.add(new IndicesShardStoresResponse.StoreStatus(response.getNode(), response.version(), allocation, response.storeException()));
            }
        }
        CollectionUtil.timSort(storeStatuses);
        indexShardsBuilder.put(fetchResponse.shardId.id(), storeStatuses);
        indicesStoreStatusesBuilder.put(fetchResponse.shardId.getIndex(), indexShardsBuilder.build());
        for (FailedNodeException failure : fetchResponse.failures) {
            failureBuilder.add(new IndicesShardStoresResponse.Failure(failure.nodeId(), fetchResponse.shardId.getIndex(), fetchResponse.shardId.id(), failure.getCause()));
        }
    }
    listener.onResponse(new IndicesShardStoresResponse(indicesStoreStatusesBuilder.build(), Collections.unmodifiableList(failureBuilder)));
}
 
Example #11
Source File: SnapshotsService.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a list of currently running snapshots from repository sorted by snapshot creation date
 *
 * @param repositoryName repository name
 * @return list of snapshots
 */
public List<Snapshot> currentSnapshots(String repositoryName) {
    List<Snapshot> snapshotList = new ArrayList<>();
    List<SnapshotsInProgress.Entry> entries = currentSnapshots(repositoryName, null);
    for (SnapshotsInProgress.Entry entry : entries) {
        snapshotList.add(inProgressSnapshot(entry));
    }
    CollectionUtil.timSort(snapshotList);
    return Collections.unmodifiableList(snapshotList);
}
 
Example #12
Source File: CommitPoints.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public CommitPoints(List<CommitPoint> commitPoints) {
    CollectionUtil.introSort(commitPoints, new Comparator<CommitPoint>() {
        @Override
        public int compare(CommitPoint o1, CommitPoint o2) {
            return (o2.version() < o1.version() ? -1 : (o2.version() == o1.version() ? 0 : 1));
        }
    });
    this.commitPoints = Collections.unmodifiableList(new ArrayList<>(commitPoints));
}
 
Example #13
Source File: ElectMasterService.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
private List<DiscoveryNode> sortedMasterNodes(Iterable<DiscoveryNode> nodes) {
    List<DiscoveryNode> possibleNodes = CollectionUtils.iterableAsArrayList(nodes);
    if (possibleNodes.isEmpty()) {
        return null;
    }
    // clean non master nodes
    for (Iterator<DiscoveryNode> it = possibleNodes.iterator(); it.hasNext(); ) {
        DiscoveryNode node = it.next();
        if (!node.masterNode()) {
            it.remove();
        }
    }
    CollectionUtil.introSort(possibleNodes, nodeComparator);
    return possibleNodes;
}
 
Example #14
Source File: GroupShardsIterator.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * Constructs a enw GroupShardsIterator from the given list.
 */
public GroupShardsIterator(List<ShardIterator> iterators) {
    CollectionUtil.timSort(iterators);
    this.iterators = iterators;
}
 
Example #15
Source File: RoutingNodes.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public void sort(Comparator<ShardRouting> comparator) {
    CollectionUtil.timSort(unassigned, comparator);
}
 
Example #16
Source File: PrimaryShardAllocator.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * Builds a list of nodes and version
 */
NodesAndVersions buildNodesAndVersions(ShardRouting shard, boolean recoveryOnAnyNode, Set<String> ignoreNodes,
                                       AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShards.NodeGatewayStartedShards> shardState) {
    final Map<DiscoveryNode, Tuple<Long, Long>> nodesWithVersion = Maps.newHashMap();
    int numberOfAllocationsFound = 0;
    long highestVersion = -1;
    for (TransportNodesListGatewayStartedShards.NodeGatewayStartedShards nodeShardState : shardState.getData().values()) {
        long version = nodeShardState.version();
        long numDocs = nodeShardState.numDocs();
        DiscoveryNode node = nodeShardState.getNode();

        if (ignoreNodes.contains(node.id())) {
            continue;
        }

        // -1 version means it does not exists, which is what the API returns, and what we expect to
        if (nodeShardState.storeException() == null) {
            logger.trace("[{}] on node [{}] has version [{}] of shard, numDocs [{}]", shard, nodeShardState.getNode(), version, numDocs);
        } else {
            // when there is an store exception, we disregard the reported version and assign it as -1 (same as shard does not exist)
            logger.trace("[{}] on node [{}] has version [{}], numDocs [{}] but the store can not be opened, treating as version -1", nodeShardState.storeException(), shard, nodeShardState.getNode(), version, numDocs);
            version = -1;
        }

        if (recoveryOnAnyNode) {
            numberOfAllocationsFound++;
            if (version > highestVersion) {
                highestVersion = version;
            }
            // We always put the node without clearing the map
            nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs));
        } else if (version != -1) {
            numberOfAllocationsFound++;
            // If we've found a new "best" candidate, clear the
            // current candidates and add it
            if (version > highestVersion) {
                highestVersion = version;
                nodesWithVersion.clear();
                nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs));
            } else if (version == highestVersion) {
                // If the candidate is the same, add it to the
                // list, but keep the current candidate
                nodesWithVersion.put(node, new Tuple<Long, Long>(version, numDocs));
            }
        }
    }
    // Now that we have a map of nodes to versions along with the
    // number of allocations found (and not ignored), we need to sort
    // it so the node with the highest version is at the beginning
    List<DiscoveryNode> nodesWithHighestVersion = new ArrayList<>();
    nodesWithHighestVersion.addAll(nodesWithVersion.keySet());
    CollectionUtil.timSort(nodesWithHighestVersion, new Comparator<DiscoveryNode>() {
        @Override
        public int compare(DiscoveryNode o1, DiscoveryNode o2) {
            int compareResult = Long.compare(nodesWithVersion.get(o2).v1(), nodesWithVersion.get(o1).v1());
            if (compareResult == 0) {
                return Long.compare(nodesWithVersion.get(o2).v2(), nodesWithVersion.get(o1).v2());
            } else {
                return compareResult;
            }
        }
    });

    if (logger.isTraceEnabled()) {
        StringBuilder sb = new StringBuilder("[");
        for (DiscoveryNode n : nodesWithVersion.keySet()) {
            sb.append("[").append(n.getName()).append("]").append(" -> ").append(nodesWithVersion.get(n)).append(", ");
        }
        sb.append("]");
        logger.trace("{} candidates for allocation: {}", shard, sb.toString());
    }

    return new NodesAndVersions(Collections.unmodifiableList(nodesWithHighestVersion), numberOfAllocationsFound, highestVersion);
}
 
Example #17
Source File: RoutingNodes.java    From crate with Apache License 2.0 4 votes vote down vote up
public void sort(Comparator<ShardRouting> comparator) {
    nodes.ensureMutable();
    CollectionUtil.timSort(unassigned, comparator);
}
 
Example #18
Source File: GroupShardsIterator.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Constructs a enw GroupShardsIterator from the given list.
 */
public GroupShardsIterator(List<ShardIt> iterators) {
    CollectionUtil.timSort(iterators);
    this.iterators = iterators;
}
 
Example #19
Source File: AbstractScopedSettings.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Validates that the settings is valid.
 *
 * @param key                            the key of the setting to validate
 * @param settings                       the settings
 * @param validateDependencies           true if dependent settings should be validated
 * @param validateInternalOrPrivateIndex true if internal index settings should be validated
 * @throws IllegalArgumentException if the setting is invalid
 */
void validate(
        final String key, final Settings settings, final boolean validateDependencies, final boolean validateInternalOrPrivateIndex) {
    Setting<?> setting = getRaw(key);
    if (setting == null) {
        LevenshteinDistance ld = new LevenshteinDistance();
        List<Tuple<Float, String>> scoredKeys = new ArrayList<>();
        for (String k : this.keySettings.keySet()) {
            float distance = ld.getDistance(key, k);
            if (distance > 0.7f) {
                scoredKeys.add(new Tuple<>(distance, k));
            }
        }
        CollectionUtil.timSort(scoredKeys, (a,b) -> b.v1().compareTo(a.v1()));
        String msg = "unknown setting [" + key + "]";
        List<String> keys = scoredKeys.stream().map((a) -> a.v2()).collect(Collectors.toList());
        if (keys.isEmpty() == false) {
            msg += " did you mean " + (keys.size() == 1 ? "[" + keys.get(0) + "]" : "any of " + keys.toString()) + "?";
        } else {
            msg += " please check that any required plugins are installed," +
                " or check the breaking changes documentation for removed settings";
        }
        throw new IllegalArgumentException(msg);
    } else {
        Set<Setting<?>> settingsDependencies = setting.getSettingsDependencies(key);
        if (setting.hasComplexMatcher()) {
            setting = setting.getConcreteSetting(key);
        }
        if (validateDependencies && settingsDependencies.isEmpty() == false) {
            for (final Setting<?> settingDependency : settingsDependencies) {
                if (settingDependency.existsOrFallbackExists(settings) == false) {
                    final String message = String.format(
                            Locale.ROOT,
                            "missing required setting [%s] for setting [%s]",
                            settingDependency.getKey(),
                            setting.getKey());
                    throw new IllegalArgumentException(message);
                }
            }
        }

        // the only time that validateInternalOrPrivateIndex should be true is if this call is coming via the update settings API
        if (validateInternalOrPrivateIndex) {
            if (setting.isInternalIndex()) {
                throw new IllegalArgumentException(
                        "can not update internal setting [" + setting.getKey() + "]; this setting is managed via a dedicated API");
            } else if (setting.isPrivateIndex()) {
                throw new IllegalArgumentException(
                        "can not update private setting [" + setting.getKey() + "]; this setting is managed by CrateDB");
            }
        }
    }
    Iterator<? extends Setting<?>> validationDependencies = setting.getValidationDependencies();
    if (validationDependencies.hasNext()) {
        Settings previousSettings = this.lastSettingsApplied;
        Settings.Builder settingsInclDependencies = Settings.builder().put(settings);
        while (validationDependencies.hasNext()) {
            Setting<?> dependency = validationDependencies.next();
            if (!settings.hasValue(dependency.getKey()) && previousSettings.hasValue(dependency.getKey())) {
                settingsInclDependencies.copy(dependency.getKey(), previousSettings);
            }
        }
        setting.get(settingsInclDependencies.build());
    } else {
        setting.get(settings);
    }
}
 
Example #20
Source File: Suggest.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
protected void sort(Comparator<O> comparator) {
    CollectionUtil.timSort(options, comparator);
}
 
Example #21
Source File: ElectMasterService.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the given nodes sorted by likelyhood of being elected as master, most likely first.
 * Non-master nodes are not removed but are rather put in the end
 */
public List<DiscoveryNode> sortByMasterLikelihood(Iterable<DiscoveryNode> nodes) {
    ArrayList<DiscoveryNode> sortedNodes = CollectionUtils.iterableAsArrayList(nodes);
    CollectionUtil.introSort(sortedNodes, nodeComparator);
    return sortedNodes;
}