org.elasticsearch.cluster.metadata.IndexMetaData Java Examples

public DocIndexMetaData(Functions functions, IndexMetaData metaData, TableIdent ident) throws IOException {
    this.functions = functions;
    this.ident = ident;
    this.metaData = metaData;
    this.isAlias = !metaData.getIndex().equals(ident.indexName());
    this.numberOfShards = metaData.getNumberOfShards();
    Settings settings = metaData.getSettings();
    this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
    this.aliases = ImmutableSet.copyOf(metaData.getAliases().keys().toArray(String.class));
    this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
    if (defaultMappingMetaData == null) {
        this.defaultMappingMap = ImmutableMap.of();
    } else {
        this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
    }
    this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);

    Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
    indicesMap = getNested(metaMap, "indices", ImmutableMap.<String, Object>of());
    partitionedByList = getNested(metaMap, "partitioned_by", ImmutableList.<List<String>>of());
    generatedColumns = getNested(metaMap, "generated_columns", ImmutableMap.<String, String>of());
}
 

private void applySettings(ClusterChangedEvent event) {
    if (!event.metaDataChanged()) {
        return;
    }
    for (IndexMetaData indexMetaData : event.state().metaData()) {
        if (!indicesService.hasIndex(indexMetaData.getIndex())) {
            // we only create / update here
            continue;
        }
        // if the index meta data didn't change, no need check for refreshed settings
        if (!event.indexMetaDataChanged(indexMetaData)) {
            continue;
        }
        String index = indexMetaData.getIndex();
        IndexService indexService = indicesService.indexService(index);
        if (indexService == null) {
            // already deleted on us, ignore it
            continue;
        }
        IndexSettingsService indexSettingsService = indexService.injector().getInstance(IndexSettingsService.class);
        indexSettingsService.refreshSettings(indexMetaData.getSettings());
    }
}
 

public Builder addBlocks(IndexMetaData indexMetaData) {
    String indexName = indexMetaData.getIndex().getName();
    if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
        addIndexBlock(indexName, IndexMetaData.INDEX_CLOSED_BLOCK);
    }
    if (IndexMetaData.INDEX_READ_ONLY_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_READ_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_WRITE_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_METADATA_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
    }
    return this;
}
 

/**
 * Check if any of the indices to be closed are currently being restored from a snapshot and fail closing if such an index
 * is found as closing an index that is being restored makes the index unusable (it cannot be recovered).
 */
public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaData> indices) {
    RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE);
    if (restore != null) {
        Set<Index> indicesToFail = null;
        for (RestoreInProgress.Entry entry : restore.entries()) {
            for (ObjectObjectCursor<ShardId, RestoreInProgress.ShardRestoreStatus> shard : entry.shards()) {
                if (!shard.value.state().completed()) {
                    IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex());
                    if (indexMetaData != null && indices.contains(indexMetaData)) {
                        if (indicesToFail == null) {
                            indicesToFail = new HashSet<>();
                        }
                        indicesToFail.add(shard.key.getIndex());
                    }
                }
            }
        }
        if (indicesToFail != null) {
            throw new IllegalArgumentException("Cannot close indices that are being restored: " + indicesToFail);
        }
    }
}
 

/**
 * {@inheritDoc}
 */
@Override
protected void doStart() {
    this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
    indexShardRepository.initialize(blobStore(), basePath(), chunkSize(), snapshotRateLimiter, restoreRateLimiter, this, isCompress());

    ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(settings);
    globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher, isCompress());
    globalMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher);

    indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher, isCompress());
    indexMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher);

    snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher, isCompress());
    snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher);
}
 

@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    cause = in.readString();
    name = in.readString();
    template = in.readString();
    order = in.readInt();
    create = in.readBoolean();
    settings = readSettingsFromStream(in);
    int size = in.readVInt();
    for (int i = 0; i < size; i++) {
        mappings.put(in.readString(), in.readString());
    }
    int customSize = in.readVInt();
    for (int i = 0; i < customSize; i++) {
        String type = in.readString();
        IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
        customs.put(type, customIndexMetaData);
    }
    int aliasesSize = in.readVInt();
    for (int i = 0; i < aliasesSize; i++) {
        aliases.add(Alias.read(in));
    }
}
 

private static IndexMetaData.Builder getIndexMetaData(String indexName,
                                                      Settings settings,
                                                      @Nullable Map<String, Object> mapping,
                                                      Version smallestNodeVersion) throws IOException {
    Settings.Builder builder = Settings.builder()
        .put(settings)
        .put(SETTING_VERSION_CREATED, smallestNodeVersion)
        .put(SETTING_CREATION_DATE, Instant.now().toEpochMilli())
        .put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());

    Settings indexSettings = builder.build();
    IndexMetaData.Builder metaBuilder = IndexMetaData.builder(indexName)
        .settings(indexSettings);
    if (mapping != null) {
        metaBuilder.putMapping(new MappingMetaData(
            Constants.DEFAULT_MAPPING_TYPE,
            mapping));
    }

    return metaBuilder;
}
 

/**
 * Returns list of indices with missing shards, and list of indices that are closed
 *
 * @param shards list of shard statuses
 * @return list of failed and closed indices
 */
private static Tuple<Set<String>, Set<String>> indicesWithMissingShards(
    ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards, MetaData metaData) {
    Set<String> missing = new HashSet<>();
    Set<String> closed = new HashSet<>();
    for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards) {
        if (entry.value.state() == ShardState.MISSING) {
            if (metaData.hasIndex(entry.key.getIndex().getName()) &&
                metaData.getIndexSafe(entry.key.getIndex()).getState() == IndexMetaData.State.CLOSE) {
                closed.add(entry.key.getIndex().getName());
            } else {
                missing.add(entry.key.getIndex().getName());
            }
        }
    }
    return new Tuple<>(missing, closed);
}
 

/**
 * Loads the current meta state for each index in the new cluster state and checks if it has to be persisted.
 * Each index state that should be written to disk will be returned. This is only run for data only nodes.
 * It will return only the states for indices that actually have a shard allocated on the current node.
 *
 * @param previouslyWrittenIndices    A list of indices for which the state was already written before
 * @param potentiallyUnwrittenIndices The list of indices for which state should potentially be written
 * @param previousMetaData            The last meta data we know of. meta data for all indices in previouslyWrittenIndices list is persisted now
 * @param newMetaData                 The new metadata
 * @return iterable over all indices states that should be written to disk
 */
public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(ImmutableSet<String> previouslyWrittenIndices, Set<String> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
    List<GatewayMetaState.IndexMetaWriteInfo> indicesToWrite = new ArrayList<>();
    for (String index : potentiallyUnwrittenIndices) {
        IndexMetaData newIndexMetaData = newMetaData.index(index);
        IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);
        String writeReason = null;
        if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) {
            writeReason = "freshly created";
        } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) {
            writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]";
        }
        if (writeReason != null) {
            indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason));
        }
    }
    return indicesToWrite;
}
 

/**
 * Elasticsearch 2.0 removed several deprecated features and as well as support for Lucene 3.x. This method calls
 * {@link MetaDataIndexUpgradeService} to makes sure that indices are compatible with the current version. The
 * MetaDataIndexUpgradeService might also update obsolete settings if needed.
 * Allows upgrading global custom meta data via {@link MetaDataUpgrader#customMetaDataUpgraders}
 *
 * @return input <code>metaData</code> if no upgrade is needed or an upgraded metaData
 */
static MetaData upgradeMetaData(MetaData metaData,
                                MetaDataIndexUpgradeService metaDataIndexUpgradeService,
                                MetaDataUpgrader metaDataUpgrader) {
    // upgrade index meta data
    boolean changed = false;
    final MetaData.Builder upgradedMetaData = MetaData.builder(metaData);
    for (IndexMetaData indexMetaData : metaData) {
        IndexMetaData newMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData,
                Version.CURRENT.minimumIndexCompatibilityVersion());
        changed |= indexMetaData != newMetaData;
        upgradedMetaData.put(newMetaData, false);
    }
    // upgrade global custom meta data
    if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
            upgradedMetaData::removeCustom, upgradedMetaData::putCustom)) {
        changed = true;
    }
    // upgrade current templates
    if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
        changed = true;
    }
    return changed ? upgradedMetaData.build() : metaData;
}
 

@Test
public void testBuildFromSingleIndexBlocks() throws Exception {
    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder().put(
        IndexMetaData.SETTING_READ_ONLY, true).build(), IndexMetaData.State.OPEN),
        is(Operation.READ_ONLY));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_READ, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.UPDATE, Operation.INSERT, Operation.DELETE, Operation.DROP, Operation.ALTER,
            Operation.ALTER_OPEN_CLOSE, Operation.ALTER_BLOCKS, Operation.REFRESH, Operation.OPTIMIZE, Operation.ALTER_REROUTE));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.READ, Operation.ALTER, Operation.ALTER_OPEN_CLOSE, Operation.ALTER_BLOCKS,
            Operation.SHOW_CREATE, Operation.REFRESH, Operation.OPTIMIZE, Operation.COPY_TO,
            Operation.CREATE_SNAPSHOT, Operation.ALTER_REROUTE));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_METADATA, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.READ, Operation.UPDATE, Operation.INSERT, Operation.DELETE, Operation.ALTER_BLOCKS,
            Operation.ALTER_OPEN_CLOSE, Operation.REFRESH, Operation.SHOW_CREATE, Operation.OPTIMIZE, Operation.ALTER_REROUTE));
}
 

/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            if (asNew && ignoreShards.contains(shardId)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
            }
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 

static void buildShardLevelInfo(ESLogger logger, ShardStats[] stats, HashMap<String, Long> newShardSizes, HashMap<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    MetaData meta = state.getMetaData();
    for (ShardStats s : stats) {
        IndexMetaData indexMeta = meta.index(s.getShardRouting().index());
        Settings indexSettings = indexMeta == null ? null : indexMeta.getSettings();
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        if (indexSettings != null && IndexMetaData.isIndexUsingShadowReplicas(indexSettings)) {
            // Shards on a shared filesystem should be considered of size 0
            if (logger.isTraceEnabled()) {
                logger.trace("shard: {} is using shadow replicas and will be treated as size 0", sid);
            }
            size = 0;
        }
        newShardSizes.put(sid, size);
    }
}
 

private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) {
    Path[] indexPaths = new Path[nodePaths.length];
    for (int i = 0; i < nodePaths.length; i++) {
        indexPaths[i] = nodePaths[i].resolve(uuid);
    }
    try {
        IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(LOGGER, namedXContentRegistry, indexPaths);
        return metaData.getIndex().getName();
    } catch (Exception e) {
        return "no name for uuid: " + uuid + ": " + e;
    }
}
 

public static IndexSettings newIndexSettings(final IndexMetaData indexMetaData, Setting<?>... setting) {
    Set<Setting<?>> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
    if (setting.length > 0) {
        settingSet.addAll(Arrays.asList(setting));
    }
    return new IndexSettings(indexMetaData, Settings.EMPTY, new IndexScopedSettings(Settings.EMPTY, settingSet));
}
 

private Decision doDecide(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
                          BiPredicate<Integer, Integer> decider) {
    IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
    final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limits are disabled: [index: %d, cluster: %d] <= 0",
                indexShardLimit, clusterShardLimit);
    }

    int indexShardCount = 0;
    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
        if (nodeShard.index().equals(shardRouting.index())) {
            indexShardCount++;
        }
    }

    if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
            nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
    }
    if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
            indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
    }
    return allocation.decision(Decision.YES, NAME,
        "the shard count [%d] for this node is under the index limit [%d] and cluster level node limit [%d]",
        nodeShardCount, indexShardLimit, clusterShardLimit);
}
 

@Test
public void testSchemaWithNotNullGeneratedColumn() throws Exception {
    XContentBuilder builder = XContentFactory.jsonBuilder()
        .startObject()
            .startObject("_meta")
                .startObject("generated_columns")
                    .field("week", "date_trunc('week', ts)")
                .endObject()
                .startObject("constraints")
                    .array("not_null", "week")
                .endObject()
            .endObject()
            .startObject("properties")
                .startObject("ts").field("type", "date").endObject()
                .startObject("week").field("type", "long").endObject()
            .endObject()
        .endObject();

    IndexMetaData metaData = getIndexMetaData("test1", builder);
    DocIndexMetaData md = newMeta(metaData, "test1");

    assertThat(md.columns().size(), is(2));
    Reference week = md.references().get(new ColumnIdent("week"));
    assertThat(week, Matchers.notNullValue());
    assertThat(week.isNullable(), is(false));
    assertThat(week, instanceOf(GeneratedReference.class));
    assertThat(((GeneratedReference) week).formattedGeneratedExpression(), is("date_trunc('week', ts)"));
    assertThat(((GeneratedReference) week).generatedExpression(), isFunction("date_trunc", isLiteral("week"), isReference("ts")));
    assertThat(((GeneratedReference) week).referencedReferences(), contains(isReference("ts")));
}
 

public static int calculateScaledShardId(IndexMetaData indexMetaData, String effectiveRouting, int partitionOffset) {
    final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset;

    // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size
    // of original index to hash documents
    return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
}
 

public void allocateDangled(Collection<IndexMetaData> indices, final Listener listener) {
    ClusterState clusterState = clusterService.state();
    DiscoveryNode masterNode = clusterState.nodes().getMasterNode();
    if (masterNode == null) {
        listener.onFailure(new MasterNotDiscoveredException("no master to send allocate dangled request"));
        return;
    }
    AllocateDangledRequest request = new AllocateDangledRequest(clusterService.localNode(),
        indices.toArray(new IndexMetaData[indices.size()]));
    transportService.sendRequest(masterNode, ACTION_NAME, request, new TransportResponseHandler<AllocateDangledResponse>() {
        @Override
        public AllocateDangledResponse read(StreamInput in) throws IOException {
            return new AllocateDangledResponse(in);
        }

        @Override
        public void handleResponse(AllocateDangledResponse response) {
            listener.onResponse(response);
        }

        @Override
        public void handleException(TransportException exp) {
            listener.onFailure(exp);
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }
    });
}
 

protected DocIndexMetaData merge(DocIndexMetaData other,
                                 TransportPutIndexTemplateAction transportPutIndexTemplateAction,
                                 boolean thisIsCreatedFromTemplate) throws IOException {
    if (schemaEquals(other)) {
        return this;
    } else if (thisIsCreatedFromTemplate) {
        if (this.references.size() < other.references.size()) {
            // this is older, update template and return other
            // settings in template are always authoritative for table information about
            // number_of_shards and number_of_replicas
            updateTemplate(other, transportPutIndexTemplateAction, this.metaData.getSettings());
            // merge the new mapping with the template settings
            return new DocIndexMetaData(
                    functions,
                    IndexMetaData.builder(other.metaData).settings(this.metaData.getSettings()).build(),
                    other.ident).build();
        } else if (references().size() == other.references().size() &&
                   !references().keySet().equals(other.references().keySet())) {
            XContentHelper.update(defaultMappingMap, other.defaultMappingMap, false);
            // update the template with new information
            updateTemplate(this, transportPutIndexTemplateAction, this.metaData.getSettings());
            return this;
        }
        // other is older, just return this
        return this;
    } else {
        throw new TableAliasSchemaException(other.ident.name());
    }
}
 

public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
    for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) {
        final MetaData metaData = allocation.metaData();
        final IndexMetaData indexMetaData = metaData.index(shardRouting.index());
        shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings());
    }
}
 

public Builder addAsFromCloseToOpen(IndexMetaData indexMetaData) {
    if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
        IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
                .initializeAsFromCloseToOpen(indexMetaData);
        add(indexRoutingBuilder);
    }
    return this;
}
 

public static MapperService newMapperService(Settings settings, Client client) {
    Settings indexSettings = Settings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("path.home", System.getProperty("path.home"))
            .put("client.type", "node")
            .put(settings)
            .build();
    Index index = new Index("test");
    Injector parentInjector = new ModulesBuilder()
            .add(new SettingsModule(indexSettings),
            new EnvironmentModule(new Environment(indexSettings)))
            .createInjector();
    AnalysisModule analysisModule = new AnalysisModule(indexSettings,
            parentInjector.getInstance(IndicesAnalysisService.class));
    new AnalysisBaseformPlugin(settings).onModule(analysisModule);
    Injector injector = new ModulesBuilder().add(new IndexSettingsModule(index, indexSettings),
            new IndexNameModule(index),
            analysisModule)
            .createChildInjector(parentInjector);
    AnalysisService analysisService = injector.getInstance(AnalysisService.class);
    SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, indexSettings);
    Map<String, Mapper.TypeParser> mappers = registerBuiltInMappers();
    Map<String, MetadataFieldMapper.TypeParser> metadataMappers = registerBuiltInMetadataMappers();
    MapperRegistry mapperRegistry = new MapperRegistry(mappers, metadataMappers);
    return new MapperService(new Index("test"),
            indexSettings,
            analysisService,
            similarityLookupService,
            null,
            mapperRegistry);
}
 

@Test
public void testSchemaWithNotNullNestedColumns() throws Exception {
    XContentBuilder builder = XContentFactory.jsonBuilder()
    .startObject()
        .startObject(Constants.DEFAULT_MAPPING_TYPE)
            .startObject("_meta")
                .startObject("constraints")
                    .array("not_null", "nested.level1", "nested.level1.level2")
                .endObject()
            .endObject()
            .startObject("properties")
                .startObject("nested")
                    .field("type", "object")
                        .startObject("properties")
                            .startObject("level1")
                                .field("type", "object")
                                .startObject("properties")
                                    .startObject("level2")
                                        .field("type", "string")
                                    .endObject()
                                .endObject()
                            .endObject()
                         .endObject()
                .endObject()
            .endObject()
        .endObject()
    .endObject();
    IndexMetaData metaData = getIndexMetaData("test_notnull_columns", builder);
    DocIndexMetaData md = newMeta(metaData, "test_notnull_columns");

    ColumnIdent level1 = new ColumnIdent("nested", "level1");
    ColumnIdent level2 = new ColumnIdent("nested", Arrays.asList("level1", "level2"));
    assertThat(md.notNullColumns(), containsInAnyOrder(level1, level2));
    assertThat(md.references().get(level1).isNullable(), is(false));
    assertThat(md.references().get(level2).isNullable(), is(false));
}
 

/**
 * Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and disable_purge to false
 */
private List<IndexShard> getShardsToPurge() {
    List<IndexShard> shardsToPurge = new ArrayList<>();
    MetaData metaData = clusterService.state().metaData();
    for (IndexService indexService : indicesService) {
        // check the value of disable_purge for this index
        IndexMetaData indexMetaData = metaData.index(indexService.index().name());
        if (indexMetaData == null) {
            continue;
        }
        boolean disablePurge = indexMetaData.getSettings().getAsBoolean(INDEX_TTL_DISABLE_PURGE, false);
        if (disablePurge) {
            continue;
        }

        // check if ttl is enabled for at least one type of this index
        boolean hasTTLEnabled = false;
        for (String type : indexService.mapperService().types()) {
            DocumentMapper documentType = indexService.mapperService().documentMapper(type);
            if (documentType.TTLFieldMapper().enabled()) {
                hasTTLEnabled = true;
                break;
            }
        }
        if (hasTTLEnabled) {
            for (IndexShard indexShard : indexService) {
                if (indexShard.state() == IndexShardState.STARTED && indexShard.routingEntry().primary() && indexShard.routingEntry().started()) {
                    shardsToPurge.add(indexShard);
                }
            }
        }
    }
    return shardsToPurge;
}
 

@Override
public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) {
    final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability();

    final boolean updateIndexMetaData = indexSettings.updateIndexMetaData(newIndexMetaData);

    if (Assertions.ENABLED
            && currentIndexMetaData != null
            && currentIndexMetaData.getCreationVersion().onOrAfter(Version.ES_V_6_5_1)) {
        final long currentSettingsVersion = currentIndexMetaData.getSettingsVersion();
        final long newSettingsVersion = newIndexMetaData.getSettingsVersion();
        if (currentSettingsVersion == newSettingsVersion) {
            assert updateIndexMetaData == false;
        } else {
            assert updateIndexMetaData;
            assert currentSettingsVersion < newSettingsVersion :
                    "expected current settings version [" + currentSettingsVersion + "] "
                            + "to be less than new settings version [" + newSettingsVersion + "]";
        }
    }

    if (updateIndexMetaData) {
        for (final IndexShard shard : this.shards.values()) {
            try {
                shard.onSettingsChanged();
            } catch (Exception e) {
                logger.warn(
                    () -> new ParameterizedMessage(
                        "[{}] failed to notify shard about setting change", shard.shardId().id()), e);
            }
        }
        if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
            rescheduleRefreshTasks();
        }
        final Translog.Durability durability = indexSettings.getTranslogDurability();
        if (durability != oldTranslogDurability) {
            rescheduleFsyncTask(durability);
        }
    }
}
 

private void assertMappingVersion(
        final IndexMetaData currentIndexMetaData,
        final IndexMetaData newIndexMetaData,
        final Map<String, DocumentMapper> updatedEntries) {
    if (Assertions.ENABLED
            && currentIndexMetaData != null
            && currentIndexMetaData.getCreationVersion().onOrAfter(Version.ES_V_6_5_1)) {
        if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) {
            // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same
            assert updatedEntries.isEmpty() : updatedEntries;
            for (final ObjectCursor<MappingMetaData> mapping : newIndexMetaData.getMappings().values()) {
                final CompressedXContent currentSource = currentIndexMetaData.mapping(mapping.value.type()).source();
                final CompressedXContent newSource = mapping.value.source();
                assert currentSource.equals(newSource) :
                        "expected current mapping [" + currentSource + "] for type [" + mapping.value.type() + "] "
                                + "to be the same as new mapping [" + newSource + "]";
            }
        } else {
            // if the mapping version is changed, it should increase, there should be updates, and the mapping should be different
            final long currentMappingVersion = currentIndexMetaData.getMappingVersion();
            final long newMappingVersion = newIndexMetaData.getMappingVersion();
            assert currentMappingVersion < newMappingVersion :
                    "expected current mapping version [" + currentMappingVersion + "] "
                            + "to be less than new mapping version [" + newMappingVersion + "]";
            assert updatedEntries.isEmpty() == false;
            for (final DocumentMapper documentMapper : updatedEntries.values()) {
                final MappingMetaData currentMapping = currentIndexMetaData.mapping(documentMapper.type());
                if (currentMapping != null) {
                    final CompressedXContent currentSource = currentMapping.source();
                    final CompressedXContent newSource = documentMapper.mappingSource();
                    assert currentSource.equals(newSource) == false :
                            "expected current mapping [" + currentSource + "] for type [" + documentMapper.type() + "] " +
                                    "to be different than new mapping";
                }
            }
        }
    }
}
 

@Override
public CompletableFuture<?> computeResult() {
    HashSet<String> fqTables = new HashSet<>();
    for (ObjectCursor<IndexMetaData> cursor : clusterService.state()
        .metaData()
        .indices()
        .values()) {

        if (cursor.value.getCreationVersion().major < org.elasticsearch.Version.CURRENT.major) {
            fqTables.add(RelationName.fqnFromIndexName(cursor.value.getIndex().getName()));
        }
    }
    tablesNeedUpgrade = fqTables;
    return CompletableFuture.completedFuture(fqTables);
}
 

@Test
public void testNumberOfShardsRequestedNotAFactorOfCurrentIsNotSupported() {
    IndexMetaData indexMetaData = IndexMetaData.builder("t1")
        .settings(baseIndexSettings())
        .build();

    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("Requested number of shards: <3> needs to be a factor of the current one: <5>");
    AlterTableOperation.validateNumberOfShardsForResize(indexMetaData, 3);
}
 

/**
 * Return the {@link Version} of Elasticsearch that has been used to create an index given its settings.
 *
 * @throws IllegalStateException if the given index settings doesn't contain a value for the key {@value IndexMetaData#SETTING_VERSION_CREATED}
 */
public static Version indexCreated(Settings indexSettings) {
    final Version indexVersion = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null);
    if (indexVersion == null) {
        throw new IllegalStateException("[" + IndexMetaData.SETTING_VERSION_CREATED + "] is not present in the index settings for index with uuid: [" + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID) + "]");
    }
    return indexVersion;
}