com.carrotsearch.hppc.cursors.ObjectCursor Java Examples

The following examples show how to use com.carrotsearch.hppc.cursors.ObjectCursor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ReplicaShardAllocator.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Can the shard be allocated on at least one node based on the allocation deciders.
 */
private boolean canBeAllocatedToAtLeastOneNode(ShardRouting shard, RoutingAllocation allocation) {
    for (ObjectCursor<DiscoveryNode> cursor : allocation.nodes().dataNodes().values()) {
        RoutingNode node = allocation.routingNodes().node(cursor.value.id());
        if (node == null) {
            continue;
        }
        // if we can't allocate it on a node, ignore it, for example, this handles
        // cases for only allocating a replica after a primary
        Decision decision = allocation.deciders().canAllocate(shard, node, allocation);
        if (decision.type() == Decision.Type.YES) {
            return true;
        }
    }
    return false;
}
 
Example #2
Source File: IndicesAliasesRequest.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public String[] concreteAliases(MetaData metaData, String concreteIndex) {
    if (expandAliasesWildcards()) {
        //for DELETE we expand the aliases
        String[] indexAsArray = {concreteIndex};
        ImmutableOpenMap<String, List<AliasMetaData>> aliasMetaData = metaData.findAliases(aliases, indexAsArray);
        List<String> finalAliases = new ArrayList<>();
        for (ObjectCursor<List<AliasMetaData>> curAliases : aliasMetaData.values()) {
            for (AliasMetaData aliasMeta: curAliases.value) {
                finalAliases.add(aliasMeta.alias());
            }
        }
        return finalAliases.toArray(new String[finalAliases.size()]);
    } else {
        //for add we just return the current aliases
        return aliases;
    }
}
 
Example #3
Source File: DiscoveryNodes.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that a node can be safely added to this node collection.
 *
 * @return null if all is OK or an error message explaining why a node can not be added.
 *
 * Note: if this method returns a non-null value, calling {@link #add(DiscoveryNode)} will fail with an
 * exception
 */
private String validateAdd(DiscoveryNode node) {
    for (ObjectCursor<DiscoveryNode> cursor : nodes.values()) {
        final DiscoveryNode existingNode = cursor.value;
        if (node.getAddress().equals(existingNode.getAddress()) &&
            node.getId().equals(existingNode.getId()) == false) {
            return "can't add node " + node + ", found existing node " + existingNode + " with same address";
        }
        if (node.getId().equals(existingNode.getId()) &&
            node.equals(existingNode) == false) {
            return "can't add node " + node + ", found existing node " + existingNode
                + " with the same id but is a different node instance";
        }
    }
    return null;
}
 
Example #4
Source File: ClusterChangedEvent.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the indices created in this event
 */
public List<String> indicesCreated() {
    if (!metaDataChanged()) {
        return Collections.emptyList();
    }
    List<String> created = null;
    for (ObjectCursor<String> cursor : state.metaData().indices().keys()) {
        String index = cursor.value;
        if (!previousState.metaData().hasIndex(index)) {
            if (created == null) {
                created = new ArrayList<>();
            }
            created.add(index);
        }
    }
    return created == null ? Collections.<String>emptyList() : created;
}
 
Example #5
Source File: ClusterChangedEvent.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the indices deleted in this event
 */
public List<String> indicesDeleted() {
    // If the new cluster state has a new cluster UUID, the likely scenario is that a node was elected
    // master that has had its data directory wiped out, in which case we don't want to delete the indices and lose data;
    // rather we want to import them as dangling indices instead.  So we check here if the cluster UUID differs from the previous
    // cluster UUID, in which case, we don't want to delete indices that the master erroneously believes shouldn't exist.
    // See test DiscoveryWithServiceDisruptionsIT.testIndicesDeleted()
    // See discussion on https://github.com/elastic/elasticsearch/pull/9952 and
    // https://github.com/elastic/elasticsearch/issues/11665
    if (metaDataChanged() == false || isNewCluster()) {
        return Collections.emptyList();
    }
    List<String> deleted = null;
    for (ObjectCursor<String> cursor : previousState.metaData().indices().keys()) {
        String index = cursor.value;
        if (!state.metaData().hasIndex(index)) {
            if (deleted == null) {
                deleted = new ArrayList<>();
            }
            deleted.add(index);
        }
    }
    return deleted == null ? Collections.<String>emptyList() : deleted;
}
 
Example #6
Source File: ClusterChangedEvent.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the indices created in this event
 */
public List<String> indicesCreated() {
    if (!metaDataChanged()) {
        return Collections.emptyList();
    }
    List<String> created = null;
    for (ObjectCursor<String> cursor : state.metaData().indices().keys()) {
        String index = cursor.value;
        if (!previousState.metaData().hasIndex(index)) {
            if (created == null) {
                created = new ArrayList<>();
            }
            created.add(index);
        }
    }
    return created == null ? Collections.<String>emptyList() : created;
}
 
Example #7
Source File: ClusterChangedEvent.java    From crate with Apache License 2.0 6 votes vote down vote up
private List<Index> indicesDeletedFromClusterState() {
    // If the new cluster state has a new cluster UUID, the likely scenario is that a node was elected
    // master that has had its data directory wiped out, in which case we don't want to delete the indices and lose data;
    // rather we want to import them as dangling indices instead.  So we check here if the cluster UUID differs from the previous
    // cluster UUID, in which case, we don't want to delete indices that the master erroneously believes shouldn't exist.
    // See test DiscoveryWithServiceDisruptionsIT.testIndicesDeleted()
    // See discussion on https://github.com/elastic/elasticsearch/pull/9952 and
    // https://github.com/elastic/elasticsearch/issues/11665
    if (metaDataChanged() == false || isNewCluster()) {
        return Collections.emptyList();
    }
    List<Index> deleted = null;
    for (ObjectCursor<IndexMetaData> cursor : previousState.metaData().indices().values()) {
        IndexMetaData index = cursor.value;
        IndexMetaData current = state.metaData().index(index.getIndex());
        if (current == null) {
            if (deleted == null) {
                deleted = new ArrayList<>();
            }
            deleted.add(index.getIndex());
        }
    }
    return deleted == null ? Collections.<Index>emptyList() : deleted;
}
 
Example #8
Source File: MetaDataIndexUpgradeService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Checks the mappings for compatibility with the current version
 */
private void checkMappingsCompatibility(IndexMetaData indexMetaData) {
    Index index = new Index(indexMetaData.getIndex());
    Settings settings = indexMetaData.getSettings();
    try {
        SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings);
        // We cannot instantiate real analysis server at this point because the node might not have
        // been started yet. However, we don't really need real analyzers at this stage - so we can fake it
        try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) {
            try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService,
                    scriptService, mapperRegistry, dynamicArrayFieldMapperBuilderFactoryProvider)) {
                for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
                    MappingMetaData mappingMetaData = cursor.value;
                    mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
                }
            }
        }
    } catch (Exception ex) {
        // Wrap the inner exception so we have the index name in the exception message
        throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "], reason: [" + ex.getMessage() + "]", ex);
    }
}
 
Example #9
Source File: ImmutableOpenIntMap.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a direct iterator over the keys.
 */
public Iterator<VType> valuesIt() {
    final Iterator<ObjectCursor<VType>> iterator = map.values().iterator();
    return new Iterator<VType>() {
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public VType next() {
            return iterator.next().value;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
 
Example #10
Source File: MetaData.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
MetaData(String clusterUUID, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates, UserMetadata userMetadata, TenantMetadata tenantMetadata, ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
    this.clusterUUID = clusterUUID;
    this.version = version;
    this.transientSettings = transientSettings;
    this.persistentSettings = persistentSettings;
    this.settings = Settings.settingsBuilder().put(persistentSettings).put(transientSettings).build();
    this.indices = indices;
    this.customs = customs;
    this.templates = templates;
    this.userMetadata = userMetadata;
    this.tenantMetadata = tenantMetadata;
    int totalNumberOfShards = 0;
    int numberOfShards = 0;
    for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        totalNumberOfShards += cursor.value.getTotalNumberOfShards();
        numberOfShards += cursor.value.getNumberOfShards();
    }
    this.totalNumberOfShards = totalNumberOfShards;
    this.numberOfShards = numberOfShards;

    this.allIndices = allIndices;
    this.allOpenIndices = allOpenIndices;
    this.allClosedIndices = allClosedIndices;
    this.aliasAndIndexLookup = aliasAndIndexLookup;
}
 
Example #11
Source File: ImmutableOpenMap.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a direct iterator over the keys.
 */
public Iterator<KType> keysIt() {
    final Iterator<ObjectCursor<KType>> iterator = map.keys().iterator();
    return new Iterator<KType>() {
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public KType next() {
            return iterator.next().value;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
 
Example #12
Source File: MetaDataCreateIndexService.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(CreateIndexClusterStateUpdateRequest request, ClusterState state, IndexTemplateFilter indexTemplateFilter) throws IOException {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;
        if (indexTemplateFilter.apply(request, template)) {
            templates.add(template);
        }
    }

    CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
        @Override
        public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {
            return o2.order() - o1.order();
        }
    });
    return templates;
}
 
Example #13
Source File: ImmutableOpenMap.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Returns a direct iterator over the keys.
 */
public Iterator<VType> valuesIt() {
    final Iterator<ObjectCursor<VType>> iterator = map.values().iterator();
    return new Iterator<VType>() {
        @Override
        public boolean hasNext() {
            return iterator.hasNext();
        }

        @Override
        public VType next() {
            return iterator.next().value;
        }

        @Override
        public void remove() {
            throw new UnsupportedOperationException();
        }
    };
}
 
Example #14
Source File: SnapshotsService.java    From crate with Apache License 2.0 6 votes vote down vote up
private static boolean waitingShardsStartedOrUnassigned(SnapshotsInProgress snapshotsInProgress, ClusterChangedEvent event) {
    for (SnapshotsInProgress.Entry entry : snapshotsInProgress.entries()) {
        if (entry.state() == State.STARTED) {
            for (ObjectCursor<String> index : entry.waitingIndices().keys()) {
                if (event.indexRoutingTableChanged(index.value)) {
                    IndexRoutingTable indexShardRoutingTable = event.state().getRoutingTable().index(index.value);
                    for (ShardId shardId : entry.waitingIndices().get(index.value)) {
                        ShardRouting shardRouting = indexShardRoutingTable.shard(shardId.id()).primaryShard();
                        if (shardRouting != null && (shardRouting.started() || shardRouting.unassigned())) {
                            return true;
                        }
                    }
                }
            }
        }
    }
    return false;
}
 
Example #15
Source File: RestoreService.java    From crate with Apache License 2.0 6 votes vote down vote up
public static RestoreInProgress.State overallState(RestoreInProgress.State nonCompletedState,
                                                   ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
    boolean hasFailed = false;
    for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
        if (!status.value.state().completed()) {
            return nonCompletedState;
        }
        if (status.value.state() == RestoreInProgress.State.FAILURE) {
            hasFailed = true;
        }
    }
    if (hasFailed) {
        return RestoreInProgress.State.FAILURE;
    } else {
        return RestoreInProgress.State.SUCCESS;
    }
}
 
Example #16
Source File: FieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public void parse(FieldMapper mainField, ParseContext context) throws IOException {
    // TODO: multi fields are really just copy fields, we just need to expose "sub fields" or something that can be part of the mappings
    if (mappers.isEmpty()) {
        return;
    }

    context = context.createMultiFieldContext();

    ContentPath.Type origPathType = context.path().pathType();
    context.path().pathType(pathType);

    context.path().add(mainField.simpleName());
    for (ObjectCursor<FieldMapper> cursor : mappers.values()) {
        cursor.value.parse(context);
    }
    context.path().remove();
    context.path().pathType(origPathType);
}
 
Example #17
Source File: Elasticsearch5SearchIndex.java    From vertexium with Apache License 2.0 6 votes vote down vote up
private void loadExistingMappingIntoIndexInfo(Graph graph, IndexInfo indexInfo, String indexName) {
    indexRefreshTracker.refresh(client, indexName);
    GetMappingsResponse mapping = client.admin().indices().prepareGetMappings(indexName).get();
    for (ObjectCursor<String> mappingIndexName : mapping.getMappings().keys()) {
        ImmutableOpenMap<String, MappingMetaData> typeMappings = mapping.getMappings().get(mappingIndexName.value);
        for (ObjectCursor<String> typeName : typeMappings.keys()) {
            MappingMetaData typeMapping = typeMappings.get(typeName.value);
            Map<String, Map<String, String>> properties = getPropertiesFromTypeMapping(typeMapping);
            if (properties == null) {
                continue;
            }

            for (Map.Entry<String, Map<String, String>> propertyEntry : properties.entrySet()) {
                String rawPropertyName = propertyEntry.getKey().replace(FIELDNAME_DOT_REPLACEMENT, ".");
                loadExistingPropertyMappingIntoIndexInfo(graph, indexInfo, rawPropertyName);
            }
        }
    }
}
 
Example #18
Source File: TransportCreatePartitionsAction.java    From crate with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(CreatePartitionsRequest request, ClusterState state) {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    String firstIndex = request.indices().iterator().next();


    // note: only use the first index name to see if template matches.
    // this means
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;
        for (String pattern : template.getPatterns()) {
            if (Regex.simpleMatch(pattern, firstIndex)) {
                templates.add(template);
                break;
            }
        }
    }
    CollectionUtil.timSort(templates, (o1, o2) -> o2.order() - o1.order());
    return templates;
}
 
Example #19
Source File: BaseClient.java    From elasticsearch-helper with Apache License 2.0 6 votes vote down vote up
public String resolveMostRecentIndex(String alias) {
    if (client() == null) {
        return alias;
    }
    if (alias == null) {
        return null;
    }
    GetAliasesRequestBuilder getAliasesRequestBuilder = new GetAliasesRequestBuilder(client(), GetAliasesAction.INSTANCE);
    GetAliasesResponse getAliasesResponse = getAliasesRequestBuilder.setAliases(alias).execute().actionGet();
    Pattern pattern = Pattern.compile("^(.*?)(\\d+)$");
    Set<String> indices = new TreeSet<>(Collections.reverseOrder());
    for (ObjectCursor<String> indexName : getAliasesResponse.getAliases().keys()) {
        Matcher m = pattern.matcher(indexName.value);
        if (m.matches()) {
            if (alias.equals(m.group(1))) {
                indices.add(indexName.value);
            }
        }
    }
    return indices.isEmpty() ? alias : indices.iterator().next();
}
 
Example #20
Source File: MapperService.java    From crate with Apache License 2.0 6 votes vote down vote up
private synchronized Map<String, DocumentMapper> internalMerge(IndexMetaData indexMetaData, MergeReason reason, boolean updateAllTypes,
                                                               boolean onlyUpdateIfNeeded) {
    Map<String, CompressedXContent> map = new LinkedHashMap<>();
    for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
        MappingMetaData mappingMetaData = cursor.value;
        if (onlyUpdateIfNeeded) {
            DocumentMapper existingMapper = documentMapper(mappingMetaData.type());
            if (existingMapper == null || mappingMetaData.source().equals(existingMapper.mappingSource()) == false) {
                map.put(mappingMetaData.type(), mappingMetaData.source());
            }
        } else {
            map.put(mappingMetaData.type(), mappingMetaData.source());
        }
    }
    return internalMerge(map, reason, updateAllTypes);
}
 
Example #21
Source File: Elasticsearch7SearchIndex.java    From vertexium with Apache License 2.0 6 votes vote down vote up
private void loadExistingMappingIntoIndexInfo(Graph graph, IndexInfo indexInfo, String indexName) {
    indexRefreshTracker.refresh(client, indexName);
    GetMappingsResponse mapping = client.admin().indices().prepareGetMappings(indexName).get();
    for (ObjectCursor<String> mappingIndexName : mapping.getMappings().keys()) {
        ImmutableOpenMap<String, MappingMetaData> typeMappings = mapping.getMappings().get(mappingIndexName.value);
        for (ObjectCursor<String> typeName : typeMappings.keys()) {
            MappingMetaData typeMapping = typeMappings.get(typeName.value);
            Map<String, Map<String, String>> properties = getPropertiesFromTypeMapping(typeMapping);
            if (properties == null) {
                continue;
            }

            for (Map.Entry<String, Map<String, String>> propertyEntry : properties.entrySet()) {
                String rawPropertyName = propertyEntry.getKey().replace(FIELDNAME_DOT_REPLACEMENT, ".");
                loadExistingPropertyMappingIntoIndexInfo(graph, indexInfo, rawPropertyName);
            }
        }
    }
}
 
Example #22
Source File: TransportBulkCreateIndicesAction.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private List<IndexTemplateMetaData> findTemplates(BulkCreateIndicesRequest request,
                                                  ClusterState state,
                                                  IndexTemplateFilter indexTemplateFilter) {
    List<IndexTemplateMetaData> templates = new ArrayList<>();
    CreateIndexClusterStateUpdateRequest dummyRequest =
            new CreateIndexClusterStateUpdateRequest(request, "bulk-create", request.indices().iterator().next(), false);

    // note: only use the first index name to see if template matches.
    // this means
    for (ObjectCursor<IndexTemplateMetaData> cursor : state.metaData().templates().values()) {
        IndexTemplateMetaData template = cursor.value;

        if (indexTemplateFilter.apply(dummyRequest, template)) {
            templates.add(template);
        }
    }
    CollectionUtil.timSort(templates, new Comparator<IndexTemplateMetaData>() {
        @Override
        public int compare(IndexTemplateMetaData o1, IndexTemplateMetaData o2) {
            return o2.order() - o1.order();
        }
    });
    return templates;
}
 
Example #23
Source File: DanglingArtifactsService.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public void clusterChanged(ClusterChangedEvent event) {
    if (LOGGER.isInfoEnabled() && event.isNewCluster()) {
        for (ObjectCursor<String> key : event.state().metaData().indices().keys()) {
            for (Pattern pattern : danglingPatterns) {
                if (pattern.matcher(key.value).matches()) {
                    LOGGER.info("Dangling artifacts exist in the cluster. Use 'alter cluster gc dangling artifacts;' to remove them");
                    doStop();
                    return;
                }
            }
        }
    }
}
 
Example #24
Source File: MetaData.java    From crate with Apache License 2.0 5 votes vote down vote up
private ImmutableOpenMap<String, IndexMetaData> buildIndicesByUUIDMap() {
    ImmutableOpenMap.Builder<String, IndexMetaData> builder = ImmutableOpenMap.builder();
    for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        IndexMetaData indexMetaData = cursor.value;
        builder.put(indexMetaData.getIndexUUID(), indexMetaData);
    }
    return builder.build();
}
 
Example #25
Source File: ClusterStateUpdaters.java    From crate with Apache License 2.0 5 votes vote down vote up
static ClusterState updateRoutingTable(final ClusterState state) {
    // initialize all index routing tables as empty
    final RoutingTable.Builder routingTableBuilder = RoutingTable.builder(state.routingTable());
    for (final ObjectCursor<IndexMetaData> cursor : state.metaData().indices().values()) {
        routingTableBuilder.addAsRecovery(cursor.value);
    }
    // start with 0 based versions for routing table
    routingTableBuilder.version(0);
    return ClusterState.builder(state).routingTable(routingTableBuilder.build()).build();
}
 
Example #26
Source File: MetaDataIndexTemplateService.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Finds index templates whose index pattern matched with the given index name.
 * The result is sorted by {@link IndexTemplateMetaData#order} descending.
 */
public static List<IndexTemplateMetaData> findTemplates(MetaData metaData, String indexName) {
    final List<IndexTemplateMetaData> matchedTemplates = new ArrayList<>();
    for (ObjectCursor<IndexTemplateMetaData> cursor : metaData.templates().values()) {
        final IndexTemplateMetaData template = cursor.value;
        final boolean matched = template.patterns().stream().anyMatch(pattern -> Regex.simpleMatch(pattern, indexName));
        if (matched) {
            matchedTemplates.add(template);
        }
    }
    CollectionUtil.timSort(matchedTemplates, Comparator.comparingInt(IndexTemplateMetaData::order).reversed());
    return matchedTemplates;
}
 
Example #27
Source File: DiskThresholdDecider.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a {@link DiskUsage} for the {@link RoutingNode} using the
 * average usage of other nodes in the disk usage map.
 * @param node Node to return an averaged DiskUsage object for
 * @param usages Map of nodeId to DiskUsage for all known nodes
 * @return DiskUsage representing given node using the average disk usage
 */
DiskUsage averageUsage(RoutingNode node, ImmutableOpenMap<String, DiskUsage> usages) {
    if (usages.size() == 0) {
        return new DiskUsage(node.nodeId(), node.node().getName(), "_na_", 0, 0);
    }
    long totalBytes = 0;
    long freeBytes = 0;
    for (ObjectCursor<DiskUsage> du : usages.values()) {
        totalBytes += du.value.getTotalBytes();
        freeBytes += du.value.getFreeBytes();
    }
    return new DiskUsage(node.nodeId(), node.node().getName(), "_na_", totalBytes / usages.size(), freeBytes / usages.size());
}
 
Example #28
Source File: RoutingTable.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public void writeTo(StreamOutput out) throws IOException {
    out.writeLong(version);
    out.writeVInt(indicesRouting.size());
    for (ObjectCursor<IndexRoutingTable> index : indicesRouting.values()) {
        index.value.writeTo(out);
    }
}
 
Example #29
Source File: ReplicaShardAllocator.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Determines if the shard can be allocated on at least one node based on the allocation deciders.
 *
 * Returns the best allocation decision for allocating the shard on any node (i.e. YES if at least one
 * node decided YES, THROTTLE if at least one node decided THROTTLE, and NO if none of the nodes decided
 * YES or THROTTLE).  If in explain mode, also returns the node-level explanations as the second element
 * in the returned tuple.
 */
private Tuple<Decision, Map<String, NodeAllocationResult>> canBeAllocatedToAtLeastOneNode(ShardRouting shard,
                                                                                          RoutingAllocation allocation) {
    Decision madeDecision = Decision.NO;
    final boolean explain = allocation.debugDecision();
    Map<String, NodeAllocationResult> nodeDecisions = explain ? new HashMap<>() : null;
    for (ObjectCursor<DiscoveryNode> cursor : allocation.nodes().getDataNodes().values()) {
        RoutingNode node = allocation.routingNodes().node(cursor.value.getId());
        if (node == null) {
            continue;
        }
        // if we can't allocate it on a node, ignore it, for example, this handles
        // cases for only allocating a replica after a primary
        Decision decision = allocation.deciders().canAllocate(shard, node, allocation);
        if (decision.type() == Decision.Type.YES && madeDecision.type() != Decision.Type.YES) {
            if (explain) {
                madeDecision = decision;
            } else {
                return Tuple.tuple(decision, nodeDecisions);
            }
        } else if (madeDecision.type() == Decision.Type.NO && decision.type() == Decision.Type.THROTTLE) {
            madeDecision = decision;
        }
        if (explain) {
            nodeDecisions.put(node.nodeId(), new NodeAllocationResult(node.node(), null, decision));
        }
    }
    return Tuple.tuple(madeDecision, nodeDecisions);
}
 
Example #30
Source File: QueryComponent.java    From elasticsearch-reindex-tool with Apache License 2.0 5 votes vote down vote up
public SearchResponse prepareSearchScrollRequest() {
  // find out how many indices and shards are affected by this query to not get huge result sets when there are very many indices affected by the name, e.g. when wildcards are used
  // otherwise we regularly run into OOMs when a query goes against a large number of indices
  // I did not find a better way to find out the number of shards than to query a list of indices and for each index query the number of shards via the settings
  GetSettingsResponse getSettingsResponse = client.admin().indices().getSettings(new GetSettingsRequest().indices(dataPointer.getIndexName())).actionGet();
  int numShards = 0, numIndices = 0;
  for(ObjectCursor<Settings> settings : getSettingsResponse.getIndexToSettings().values()) {
    numShards += settings.value.getAsInt("index.number_of_shards", 0);
    numIndices++;
  }

  int sizePerShard = (int)Math.ceil((double)SCROLL_SHARD_LIMIT/numShards);
  logger.info("Found " + numIndices + " indices and " + numShards + " shards matching the index-pattern, thus setting the sizePerShard to " + sizePerShard);

  SearchRequestBuilder searchRequestBuilder = client.prepareSearch(dataPointer.getIndexName())
      .setTypes(dataPointer.getTypeName())
      .setSearchType(SearchType.SCAN)
      .addFields("_ttl", "_source")
      .setScroll(new TimeValue(SCROLL_TIME_LIMIT))
      .setSize(sizePerShard);

  if (!Strings.isNullOrEmpty(query.getQuery())) {
    searchRequestBuilder.setQuery(query.getQuery());
  }
  if (!Strings.isNullOrEmpty(query.getSortField())) {
    searchRequestBuilder.addSort(new FieldSortBuilder(query.getSortField()).order(query.getSortOrder()));
  }

  bound.map(resolvedBound -> boundedFilterFactory.createBoundedFilter(segmentationField.get(), resolvedBound))
      .ifPresent(searchRequestBuilder::setQuery);

  return searchRequestBuilder.execute().actionGet();
}