org.elasticsearch.common.collect.ImmutableOpenMap Java Examples

/**
 * 装载索引数据结构
 *
 * @param index
 * @param type
 * @return
 */
private String loadIndexStruct(String index, String type) {
    ClusterStateResponse response = client.admin().cluster().prepareState().execute().actionGet();
    ImmutableOpenMap<String, IndexMetaData> immutableOpenMap = response.getState().getMetaData().getIndices();
    if (immutableOpenMap != null) {
        IndexMetaData metaData = immutableOpenMap.get(index);
        if (metaData != null) {
            ImmutableOpenMap<String, MappingMetaData> mappings = metaData.getMappings();
            if (mappings != null) {
                MappingMetaData mappingMetaData = mappings.get(type);
                if (mappingMetaData != null) {
                    CompressedXContent content = mappingMetaData.source();
                    if (content != null) {
                        return content.toString();
                    }
                }
            }
        }
    }
    LOGGER.error("获取ES数据结构失败 index:" + index + "|type:" + type);
    return null;
}
 

public static RestoreInProgress.State overallState(RestoreInProgress.State nonCompletedState,
                                                   ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
    boolean hasFailed = false;
    for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
        if (!status.value.state().completed()) {
            return nonCompletedState;
        }
        if (status.value.state() == RestoreInProgress.State.FAILURE) {
            hasFailed = true;
        }
    }
    if (hasFailed) {
        return RestoreInProgress.State.FAILURE;
    } else {
        return RestoreInProgress.State.SUCCESS;
    }
}
 

@Override
public ClusterState execute(ClusterState currentState) {
    DiscoveryNodes.Builder nodesBuilder;
    nodesBuilder = DiscoveryNodes.builder(currentState.nodes());
    if (currentState.nodes().nodeExists(node.id())) {
        logger.debug("received a join request for an existing node [{}]", node);
        return currentState;
    } 
    // If this node is not in dead node list, then ignore this request
    ImmutableOpenMap<String, DiscoveryNode> deadNodes = clusterService.state().nodes().deadNodes();
    if (deadNodes.get(node.getIpPortAddress()) == null) {
        logger.warn("failed to find node [{}] in node list, ignore the join request", node);
        throw new IllegalStateException("could not find this node " + node + " from active node list and dead node list");
    }
    nodesBuilder.put(node);
    nodesBuilder.removeDeadNodeByIpPort(node);
    final ClusterState.Builder newStateBuilder = ClusterState.builder(currentState);
    newStateBuilder.nodes(nodesBuilder);
    ClusterState newState = newStateBuilder.build();
    return newState;
}
 

private static <Data> boolean applyPluginUpgraders(ImmutableOpenMap<String, Data> existingData,
                                                   UnaryOperator<Map<String, Data>> upgrader,
                                                   Consumer<String> removeData,
                                                   BiConsumer<String, Data> putData) {
    // collect current data
    Map<String, Data> existingMap = new HashMap<>();
    for (ObjectObjectCursor<String, Data> customCursor : existingData) {
        existingMap.put(customCursor.key, customCursor.value);
    }
    // upgrade global custom meta data
    Map<String, Data> upgradedCustoms = upgrader.apply(existingMap);
    if (upgradedCustoms.equals(existingMap) == false) {
        // remove all data first so a plugin can remove custom metadata or templates if needed
        existingMap.keySet().forEach(removeData);
        for (Map.Entry<String, Data> upgradedCustomEntry : upgradedCustoms.entrySet()) {
            putData.accept(upgradedCustomEntry.getKey(), upgradedCustomEntry.getValue());
        }
        return true;
    }
    return false;
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    int size = in.readVInt();
    ImmutableOpenMap.Builder<String, ImmutableOpenMap<String, MappingMetaData>> indexMapBuilder = ImmutableOpenMap.builder();
    for (int i = 0; i < size; i++) {
        String key = in.readString();
        int valueSize = in.readVInt();
        ImmutableOpenMap.Builder<String, MappingMetaData> typeMapBuilder = ImmutableOpenMap.builder();
        for (int j = 0; j < valueSize; j++) {
            typeMapBuilder.put(in.readString(), MappingMetaData.PROTO.readFrom(in));
        }
        indexMapBuilder.put(key, typeMapBuilder.build());
    }
    mappings = indexMapBuilder.build();
}
 

@Override
protected CountDownLatch updateNodeStats(ActionListener<NodesStatsResponse> listener) {
    return super.updateNodeStats(new ActionListener<>() {
        @Override
        public void onResponse(NodesStatsResponse nodesStatsResponse) {
            ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsagesBuilder = ImmutableOpenMap.builder();
            ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsagesBuilder = ImmutableOpenMap.builder();
            fillDiskUsagePerNode(
                LOGGER,
                adjustNodesStats(nodesStatsResponse.getNodes()),
                leastAvailableUsagesBuilder,
                mostAvailableUsagesBuilder
            );
            leastAvailableSpaceUsages = leastAvailableUsagesBuilder.build();
            mostAvailableSpaceUsages = mostAvailableUsagesBuilder.build();
        }

        @Override
        public void onFailure(Exception e) {
        }
    });
}
 

public RestoreInProgress(StreamInput in) throws IOException {
    Entry[] entries = new Entry[in.readVInt()];
    for (int i = 0; i < entries.length; i++) {
        Snapshot snapshot = new Snapshot(in);
        State state = State.fromValue(in.readByte());
        int indices = in.readVInt();
        List<String> indexBuilder = new ArrayList<>();
        for (int j = 0; j < indices; j++) {
            indexBuilder.add(in.readString());
        }
        ImmutableOpenMap.Builder<ShardId, ShardRestoreStatus> builder = ImmutableOpenMap.builder();
        int shards = in.readVInt();
        for (int j = 0; j < shards; j++) {
            ShardId shardId = new ShardId(in);
            ShardRestoreStatus shardState = ShardRestoreStatus.readShardRestoreStatus(in);
            builder.put(shardId, shardState);
        }
        entries[i] = new Entry(snapshot, state, Collections.unmodifiableList(indexBuilder), builder.build());
    }
    this.entries = Arrays.asList(entries);
}
 

GetIndexResponse(String[] indices, ImmutableOpenMap<String, List<IndexWarmersMetaData.Entry>> warmers,
        ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings,
        ImmutableOpenMap<String, List<AliasMetaData>> aliases, ImmutableOpenMap<String, Settings> settings) {
    this.indices = indices;
    if (warmers != null) {
        this.warmers = warmers;
    }
    if (mappings != null) {
        this.mappings = mappings;
    }
    if (aliases != null) {
        this.aliases = aliases;
    }
    if (settings != null) {
        this.settings = settings;
    }
}
 

@SuppressWarnings("unchecked")
MetaData(String clusterUUID, long version, Settings transientSettings, Settings persistentSettings, ImmutableOpenMap<String, IndexMetaData> indices, ImmutableOpenMap<String, IndexTemplateMetaData> templates, UserMetadata userMetadata, TenantMetadata tenantMetadata, ImmutableOpenMap<String, Custom> customs, String[] allIndices, String[] allOpenIndices, String[] allClosedIndices, SortedMap<String, AliasOrIndex> aliasAndIndexLookup) {
    this.clusterUUID = clusterUUID;
    this.version = version;
    this.transientSettings = transientSettings;
    this.persistentSettings = persistentSettings;
    this.settings = Settings.settingsBuilder().put(persistentSettings).put(transientSettings).build();
    this.indices = indices;
    this.customs = customs;
    this.templates = templates;
    this.userMetadata = userMetadata;
    this.tenantMetadata = tenantMetadata;
    int totalNumberOfShards = 0;
    int numberOfShards = 0;
    for (ObjectCursor<IndexMetaData> cursor : indices.values()) {
        totalNumberOfShards += cursor.value.getTotalNumberOfShards();
        numberOfShards += cursor.value.getNumberOfShards();
    }
    this.totalNumberOfShards = totalNumberOfShards;
    this.numberOfShards = numberOfShards;

    this.allIndices = allIndices;
    this.allOpenIndices = allOpenIndices;
    this.allClosedIndices = allClosedIndices;
    this.aliasAndIndexLookup = aliasAndIndexLookup;
}
 

public Routing forRandomMasterOrDataNode(RelationName relationName, DiscoveryNodes nodes) {
    DiscoveryNode localNode = nodes.getLocalNode();
    if (localNode.isMasterNode() || localNode.isDataNode()) {
        return forTableOnSingleNode(relationName, localNode.getId());
    }
    ImmutableOpenMap<String, DiscoveryNode> masterAndDataNodes = nodes.getMasterAndDataNodes();
    int randomIdx = seed % masterAndDataNodes.size();
    Iterator<DiscoveryNode> it = masterAndDataNodes.valuesIt();
    int currIdx = 0;
    while (it.hasNext()) {
        if (currIdx == randomIdx) {
            return forTableOnSingleNode(relationName, it.next().getId());
        }
        currIdx++;
    }
    throw new AssertionError("Cannot find a master or data node with given random index " + randomIdx);
}
 

public ImmutableOpenMapDiff(ImmutableOpenMap<K, T> before, ImmutableOpenMap<K, T> after,
                            KeySerializer<K> keySerializer, ValueSerializer<K, T> valueSerializer) {
    super(keySerializer, valueSerializer);
    assert after != null && before != null;

    for (ObjectCursor<K> key : before.keys()) {
        if (!after.containsKey(key.value)) {
            deletes.add(key.value);
        }
    }

    for (ObjectObjectCursor<K, T> partIter : after) {
        T beforePart = before.get(partIter.key);
        if (beforePart == null) {
            upserts.put(partIter.key, partIter.value);
        } else if (partIter.value.equals(beforePart) == false) {
            if (valueSerializer.supportsDiffableValues()) {
                diffs.put(partIter.key, valueSerializer.diff(partIter.value, beforePart));
            } else {
                upserts.put(partIter.key, partIter.value);
            }
        }
    }
}
 

/**
 * Finds the specific index aliases that match with the specified aliases directly or partially via wildcards and
 * that point to the specified concrete indices or match partially with the indices via wildcards.
 *
 * @param aliases         The names of the index aliases to find
 * @param concreteIndices The concrete indexes the index aliases must point to order to be returned.
 * @return a map of index to a list of alias metadata, the list corresponding to a concrete index will be empty if no aliases are
 * present for that index
 */
public ImmutableOpenMap<String, List<AliasMetaData>> findAliases(final String[] aliases, String[] concreteIndices) {
    assert aliases != null;
    assert concreteIndices != null;
    if (concreteIndices.length == 0) {
        return ImmutableOpenMap.of();
    }

    boolean matchAllAliases = matchAllAliases(aliases);
    ImmutableOpenMap.Builder<String, List<AliasMetaData>> mapBuilder = ImmutableOpenMap.builder();
    for (String index : concreteIndices) {
        IndexMetaData indexMetaData = indices.get(index);
        List<AliasMetaData> filteredValues = new ArrayList<>();
        for (ObjectCursor<AliasMetaData> cursor : indexMetaData.getAliases().values()) {
            AliasMetaData value = cursor.value;
            if (matchAllAliases || Regex.simpleMatch(aliases, value.alias())) {
                filteredValues.add(value);
            }
        }

        if (filteredValues.isEmpty() == false) {
            // Make the list order deterministic
            CollectionUtil.timSort(filteredValues, Comparator.comparing(AliasMetaData::alias));
            mapBuilder.put(index, Collections.unmodifiableList(filteredValues));
        }
    }
    return mapBuilder.build();
}
 

static void buildShardLevelInfo(Logger logger, ShardStats[] stats, ImmutableOpenMap.Builder<String, Long> newShardSizes,
                                ImmutableOpenMap.Builder<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    for (ShardStats s : stats) {
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        newShardSizes.put(sid, size);
    }
}
 

@Override
protected void doMasterOperation(final GetMappingsRequest request, String[] concreteIndices, final ClusterState state, final ActionListener<GetMappingsResponse> listener) {
    logger.trace("serving getMapping request based on version {}", state.version());
    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> result = state.metaData().findMappings(
            concreteIndices, request.types()
    );
    listener.onResponse(new GetMappingsResponse(result));
}
 

private String getIndexFromAlias(String alias) {
    ImmutableOpenMap<String, List<AliasMetaData>> aliasMap =
        es.getClient().admin().indices().prepareGetAliases(alias).get().getAliases();
    for (String index : aliasMap.keys().toArray(String.class)) {
        for (AliasMetaData aliasEntry : aliasMap.get(index)) {
            if (aliasEntry.getAlias().equals(alias)) {
                return index;
            }
        }
    }
    return null;
}
 

public Builder(IndexTemplateMetaData indexTemplateMetaData) {
    this.name = indexTemplateMetaData.name();
    order(indexTemplateMetaData.order());
    version(indexTemplateMetaData.version());
    patterns(indexTemplateMetaData.patterns());
    settings(indexTemplateMetaData.settings());

    mappings = ImmutableOpenMap.builder(indexTemplateMetaData.mappings());
    aliases = ImmutableOpenMap.builder(indexTemplateMetaData.aliases());
}
 

/**
 * Create data node map
 * @param numDataNodes the number of data nodes
 * @return data nodes map
 *
 * TODO: ModelManagerTests has the same method.  Refactor.
 */
public static ImmutableOpenMap<String, DiscoveryNode> createDataNodes(int numDataNodes) {
    ImmutableOpenMap.Builder<String, DiscoveryNode> dataNodes = ImmutableOpenMap.builder();
    for (int i = 0; i < numDataNodes; i++) {
        dataNodes.put("foo" + i, mock(DiscoveryNode.class));
    }
    return dataNodes.build();
}
 

public List<Map<String, Object>> getAllMapping(String index) throws IOException {
    List<Map<String, Object>> result = new ArrayList<>();
    ImmutableOpenMap<String, MappingMetaData> mappings = esClient.admin().cluster().prepareState().execute()
        .actionGet().getState().getMetaData().getIndices().get(index).getMappings();

    for (ObjectObjectCursor<String, MappingMetaData> cursor : mappings) {
        log.info("type is:{}", cursor.key);
        result.add(cursor.value.getSourceAsMap());
    }
    return result;
}
 

@Override
public ElasticsearchTable getTable(SchemaTableName tableName)
{
    String indexWildcard = tableName.getTableName();
    GetIndexRequest getIndexRequest = createGetIndexRequest(indexWildcard);
    //----es scher error --
    Thread.currentThread().setName("getTable_001");
    GetIndexResponse response = client.admin().indices()
            .getIndex(getIndexRequest).actionGet();
    if (response.getIndices() == null || response.getIndices().length == 0) {
        return null;
    }
    //TODO: es中运行index名访问时可以使用*进行匹配,所以可能会返回多个index的mapping, 因此下面需要进行mapping merge  test table = test1"*"
    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetaData>> mappings = response.getMappings();

    List<IndexResolution> resolutions;
    if (mappings.size() > 0) {
        resolutions = new ArrayList<>(mappings.size());
        for (ObjectObjectCursor<String, ImmutableOpenMap<String, MappingMetaData>> indexMappings : mappings) {
            resolutions.add(buildGetIndexResult(indexMappings.key, indexMappings.value));
        }
    }
    else {
        resolutions = emptyList();
    }

    IndexResolution indexWithMerged = merge(resolutions, indexWildcard);
    return new ElasticsearchTable(typeManager, tableName.getSchemaName(), tableName.getTableName(), indexWithMerged.get());
}
 

public SnapshotsInProgress(StreamInput in) throws IOException {
    Entry[] entries = new Entry[in.readVInt()];
    for (int i = 0; i < entries.length; i++) {
        final Snapshot snapshot = new Snapshot(in);
        final boolean includeGlobalState = in.readBoolean();
        final boolean partial = in.readBoolean();
        final State state = State.fromValue(in.readByte());
        int indices = in.readVInt();
        List<IndexId> indexBuilder = new ArrayList<>();
        for (int j = 0; j < indices; j++) {
            indexBuilder.add(new IndexId(in.readString(), in.readString()));
        }
        final long startTime = in.readLong();
        ImmutableOpenMap.Builder<ShardId, ShardSnapshotStatus> builder = ImmutableOpenMap.builder();
        final int shards = in.readVInt();
        for (int j = 0; j < shards; j++) {
            ShardId shardId = new ShardId(in);
            builder.put(shardId, new ShardSnapshotStatus(in));
        }
        final long repositoryStateId = in.readLong();
        final String failure = in.readOptionalString();
        final boolean useShardGenerations;
        if (in.getVersion().onOrAfter(SnapshotsService.SHARD_GEN_IN_REPO_DATA_VERSION)) {
            useShardGenerations = in.readBoolean();
        } else {
            useShardGenerations = false;
        }
        entries[i] = new Entry(snapshot,
            includeGlobalState,
            partial,
            state,
            Collections.unmodifiableList(indexBuilder),
            startTime,
            repositoryStateId,
            builder.build(),
            failure,
            useShardGenerations);
    }
    this.entries = Arrays.asList(entries);
}
 

public static boolean completed(ImmutableOpenMap<ShardId, RestoreInProgress.ShardRestoreStatus> shards) {
    for (ObjectCursor<RestoreInProgress.ShardRestoreStatus> status : shards.values()) {
        if (!status.value.state().completed()) {
            return false;
        }
    }
    return true;
}
 

void handleJoinRequest(final DiscoveryNode node, final ClusterState state, final JoinCallback callback) {
    if (!transportService.addressSupported(node.address().getClass())) {
        logger.warn("received a wrong address type from [{}], ignoring...", node);
    } else {
        transportService.connectToNode(node);
        try {
            // if the node already in active node list, then ignore this request
            if (clusterService.state().nodes().nodeExists(node.getId())) {
                callback.onSuccess();
                return;
            }
            // If this node is not in dead node list, then ignore this request
            ImmutableOpenMap<String, DiscoveryNode> deadNodes = clusterService.state().nodes().deadNodes();
            if (deadNodes.get(node.getIpPortAddress()) == null) {
                logger.warn("failed to find node [{}] in node list, ignore the join request", node);
                callback.onFailure(new IllegalStateException("could not find this node " + node + " from node list"));
                return;
            }
        } catch (Throwable e) {
            logger.warn("failed to validate incoming join request from node [{}]", node);
            callback.onFailure(new IllegalStateException("failure when sending a validation request to node", e));
            return;
        }
        // join task has to be immediate because if node is left then all shards is failed, this will generate a lot of tasks
        clusterService.submitStateUpdateTask("dl-disco-join(join from node[" + node + "])", new ProcessJoinsTask(Priority.IMMEDIATE, node, callback));
    }
}
 

private DiscoveryNode randomMasterOrDataNode() {
    ImmutableOpenMap<String, DiscoveryNode> masterAndDataNodes = clusterService.state().nodes().masterAndDataNodes();
    int randomIdx = random.nextInt(masterAndDataNodes.size());
    Iterator<DiscoveryNode> it = masterAndDataNodes.valuesIt();
    int currIdx = 0;
    while (it.hasNext()) {
        if (currIdx == randomIdx) {
            return it.next();
        }
        currIdx++;
    }
    throw new AssertionError(String.format(Locale.ENGLISH,
            "Cannot find a master or data node with given random index %d", randomIdx));
}
 

/**
 * Creates new restore metadata
 *
 * @param snapshot   snapshot
 * @param state      current state of the restore process
 * @param indices    list of indices being restored
 * @param shards     map of shards being restored to their current restore status
 */
public Entry(Snapshot snapshot, State state, List<String> indices, ImmutableOpenMap<ShardId, ShardRestoreStatus> shards) {
    this.snapshot = Objects.requireNonNull(snapshot);
    this.state = Objects.requireNonNull(state);
    this.indices = Objects.requireNonNull(indices);
    if (shards == null) {
        this.shards = ImmutableOpenMap.of();
    } else {
        this.shards = shards;
    }
}
 

public ClusterState(ClusterName clusterName, long version, String stateUUID, MetaData metaData, RoutingTable routingTable, DiscoveryNodes nodes, ClusterBlocks blocks, ImmutableOpenMap<String, Custom> customs, boolean wasReadFromDiff) {
    this.version = version;
    this.stateUUID = stateUUID;
    this.clusterName = clusterName;
    this.metaData = metaData;
    this.routingTable = routingTable;
    this.nodes = nodes;
    this.blocks = blocks;
    this.customs = customs;
    this.status = ClusterStateStatus.UNKNOWN;
    this.wasReadFromDiff = wasReadFromDiff;
}
 

@Test
public void showAll_atLeastOneIndexReturns() throws SqlParseException, SQLFeatureNotSupportedException, IOException {
    String query = "show *";
    GetIndexResponse getIndexResponse = runShowQuery(query);
    ImmutableOpenMap<String, ImmutableOpenMap<String, MappingMetadata>> mappings = getIndexResponse.getMappings();
    Assert.assertTrue(mappings.size() >= 1);

}
 

public IndexMetaData build() {
    ImmutableOpenMap.Builder<String, AliasMetaData> tmpAliases = aliases;
    Settings tmpSettings = settings;

    // update default mapping on the MappingMetaData
    if (mappings.containsKey(MapperService.DEFAULT_MAPPING)) {
        MappingMetaData defaultMapping = mappings.get(MapperService.DEFAULT_MAPPING);
        for (ObjectCursor<MappingMetaData> cursor : mappings.values()) {
            cursor.value.updateDefaultMapping(defaultMapping);
        }
    }

    return new IndexMetaData(index, version, state, tmpSettings, mappings.build(), tmpAliases.build(), customs.build(), indexOwnerTenantId);
}
 

public IndexTemplateMetaData(String name, int order, String template, Settings settings, ImmutableOpenMap<String, CompressedXContent> mappings,
                             ImmutableOpenMap<String, AliasMetaData> aliases, ImmutableOpenMap<String, IndexMetaData.Custom> customs, long templateOwnerTenantId) {
    this.name = name;
    this.order = order;
    this.template = template;
    this.settings = settings;
    this.mappings = mappings;
    this.aliases = aliases;
    this.customs = customs;
    this.templateOwnerTenantId = templateOwnerTenantId;
}
 

public ClusterBlocks(StreamInput in) throws IOException {
    Set<ClusterBlock> global = readBlockSet(in);
    int size = in.readVInt();
    ImmutableOpenMap.Builder<String, Set<ClusterBlock>> indicesBuilder = ImmutableOpenMap.builder(size);
    for (int j = 0; j < size; j++) {
        indicesBuilder.put(in.readString().intern(), readBlockSet(in));
    }
    this.global = global;
    this.indicesBlocks = indicesBuilder.build();
    levelHolders = generateLevelHolders(global, indicesBlocks);
}
 

@VisibleForTesting
static ClusterState addDefaultTemplate(ClusterState currentState) throws IOException {
    MetaData currentMetaData = currentState.getMetaData();
    ImmutableOpenMap<String, IndexTemplateMetaData> currentTemplates = currentMetaData.getTemplates();
    ImmutableOpenMap<String, IndexTemplateMetaData> newTemplates = createCopyWithDefaultTemplateAdded(currentTemplates);
    MetaData.Builder mdBuilder = MetaData.builder(currentMetaData).templates(newTemplates);
    return ClusterState.builder(currentState).metaData(mdBuilder).build();
}