Java Code Examples for org.elasticsearch.cluster.metadata.IndexMetaData

The following examples show how to use org.elasticsearch.cluster.metadata.IndexMetaData. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: DocIndexMetaData.java    License: Apache License 2.0 6 votes vote down vote up
public DocIndexMetaData(Functions functions, IndexMetaData metaData, TableIdent ident) throws IOException {
    this.functions = functions;
    this.ident = ident;
    this.metaData = metaData;
    this.isAlias = !metaData.getIndex().equals(ident.indexName());
    this.numberOfShards = metaData.getNumberOfShards();
    Settings settings = metaData.getSettings();
    this.numberOfReplicas = NumberOfReplicas.fromSettings(settings);
    this.aliases = ImmutableSet.copyOf(metaData.getAliases().keys().toArray(String.class));
    this.defaultMappingMetaData = this.metaData.mappingOrDefault(Constants.DEFAULT_MAPPING_TYPE);
    if (defaultMappingMetaData == null) {
        this.defaultMappingMap = ImmutableMap.of();
    } else {
        this.defaultMappingMap = this.defaultMappingMetaData.sourceAsMap();
    }
    this.tableParameters = TableParameterInfo.tableParametersFromIndexMetaData(metaData);

    Map<String, Object> metaMap = getNested(defaultMappingMap, "_meta");
    indicesMap = getNested(metaMap, "indices", ImmutableMap.<String, Object>of());
    partitionedByList = getNested(metaMap, "partitioned_by", ImmutableList.<List<String>>of());
    generatedColumns = getNested(metaMap, "generated_columns", ImmutableMap.<String, String>of());
}
 
Example 2
Source Project: Elasticsearch   Source File: IndicesClusterStateService.java    License: Apache License 2.0 6 votes vote down vote up
private void applySettings(ClusterChangedEvent event) {
    if (!event.metaDataChanged()) {
        return;
    }
    for (IndexMetaData indexMetaData : event.state().metaData()) {
        if (!indicesService.hasIndex(indexMetaData.getIndex())) {
            // we only create / update here
            continue;
        }
        // if the index meta data didn't change, no need check for refreshed settings
        if (!event.indexMetaDataChanged(indexMetaData)) {
            continue;
        }
        String index = indexMetaData.getIndex();
        IndexService indexService = indicesService.indexService(index);
        if (indexService == null) {
            // already deleted on us, ignore it
            continue;
        }
        IndexSettingsService indexSettingsService = indexService.injector().getInstance(IndexSettingsService.class);
        indexSettingsService.refreshSettings(indexMetaData.getSettings());
    }
}
 
Example 3
Source Project: crate   Source File: ClusterBlocks.java    License: Apache License 2.0 6 votes vote down vote up
public Builder addBlocks(IndexMetaData indexMetaData) {
    String indexName = indexMetaData.getIndex().getName();
    if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
        addIndexBlock(indexName, IndexMetaData.INDEX_CLOSED_BLOCK);
    }
    if (IndexMetaData.INDEX_READ_ONLY_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_READ_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_WRITE_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_WRITE_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_METADATA_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_METADATA_BLOCK);
    }
    if (IndexMetaData.INDEX_BLOCKS_READ_ONLY_ALLOW_DELETE_SETTING.get(indexMetaData.getSettings())) {
        addIndexBlock(indexName, IndexMetaData.INDEX_READ_ONLY_ALLOW_DELETE_BLOCK);
    }
    return this;
}
 
Example 4
Source Project: crate   Source File: RestoreService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Check if any of the indices to be closed are currently being restored from a snapshot and fail closing if such an index
 * is found as closing an index that is being restored makes the index unusable (it cannot be recovered).
 */
public static void checkIndexClosing(ClusterState currentState, Set<IndexMetaData> indices) {
    RestoreInProgress restore = currentState.custom(RestoreInProgress.TYPE);
    if (restore != null) {
        Set<Index> indicesToFail = null;
        for (RestoreInProgress.Entry entry : restore.entries()) {
            for (ObjectObjectCursor<ShardId, RestoreInProgress.ShardRestoreStatus> shard : entry.shards()) {
                if (!shard.value.state().completed()) {
                    IndexMetaData indexMetaData = currentState.metaData().index(shard.key.getIndex());
                    if (indexMetaData != null && indices.contains(indexMetaData)) {
                        if (indicesToFail == null) {
                            indicesToFail = new HashSet<>();
                        }
                        indicesToFail.add(shard.key.getIndex());
                    }
                }
            }
        }
        if (indicesToFail != null) {
            throw new IllegalArgumentException("Cannot close indices that are being restored: " + indicesToFail);
        }
    }
}
 
Example 5
Source Project: Elasticsearch   Source File: BlobStoreRepository.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * {@inheritDoc}
 */
@Override
protected void doStart() {
    this.snapshotsBlobContainer = blobStore().blobContainer(basePath());
    indexShardRepository.initialize(blobStore(), basePath(), chunkSize(), snapshotRateLimiter, restoreRateLimiter, this, isCompress());

    ParseFieldMatcher parseFieldMatcher = new ParseFieldMatcher(settings);
    globalMetaDataFormat = new ChecksumBlobStoreFormat<>(METADATA_CODEC, METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher, isCompress());
    globalMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_METADATA_NAME_FORMAT, MetaData.PROTO, parseFieldMatcher);

    indexMetaDataFormat = new ChecksumBlobStoreFormat<>(INDEX_METADATA_CODEC, METADATA_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher, isCompress());
    indexMetaDataLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, IndexMetaData.PROTO, parseFieldMatcher);

    snapshotFormat = new ChecksumBlobStoreFormat<>(SNAPSHOT_CODEC, SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher, isCompress());
    snapshotLegacyFormat = new LegacyBlobStoreFormat<>(LEGACY_SNAPSHOT_NAME_FORMAT, Snapshot.PROTO, parseFieldMatcher);
}
 
Example 6
Source Project: Elasticsearch   Source File: PutIndexTemplateRequest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void readFrom(StreamInput in) throws IOException {
    super.readFrom(in);
    cause = in.readString();
    name = in.readString();
    template = in.readString();
    order = in.readInt();
    create = in.readBoolean();
    settings = readSettingsFromStream(in);
    int size = in.readVInt();
    for (int i = 0; i < size; i++) {
        mappings.put(in.readString(), in.readString());
    }
    int customSize = in.readVInt();
    for (int i = 0; i < customSize; i++) {
        String type = in.readString();
        IndexMetaData.Custom customIndexMetaData = IndexMetaData.lookupPrototypeSafe(type).readFrom(in);
        customs.put(type, customIndexMetaData);
    }
    int aliasesSize = in.readVInt();
    for (int i = 0; i < aliasesSize; i++) {
        aliases.add(Alias.read(in));
    }
}
 
Example 7
Source Project: crate   Source File: SQLExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private static IndexMetaData.Builder getIndexMetaData(String indexName,
                                                      Settings settings,
                                                      @Nullable Map<String, Object> mapping,
                                                      Version smallestNodeVersion) throws IOException {
    Settings.Builder builder = Settings.builder()
        .put(settings)
        .put(SETTING_VERSION_CREATED, smallestNodeVersion)
        .put(SETTING_CREATION_DATE, Instant.now().toEpochMilli())
        .put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());

    Settings indexSettings = builder.build();
    IndexMetaData.Builder metaBuilder = IndexMetaData.builder(indexName)
        .settings(indexSettings);
    if (mapping != null) {
        metaBuilder.putMapping(new MappingMetaData(
            Constants.DEFAULT_MAPPING_TYPE,
            mapping));
    }

    return metaBuilder;
}
 
Example 8
Source Project: crate   Source File: SnapshotsService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Returns list of indices with missing shards, and list of indices that are closed
 *
 * @param shards list of shard statuses
 * @return list of failed and closed indices
 */
private static Tuple<Set<String>, Set<String>> indicesWithMissingShards(
    ImmutableOpenMap<ShardId, SnapshotsInProgress.ShardSnapshotStatus> shards, MetaData metaData) {
    Set<String> missing = new HashSet<>();
    Set<String> closed = new HashSet<>();
    for (ObjectObjectCursor<ShardId, SnapshotsInProgress.ShardSnapshotStatus> entry : shards) {
        if (entry.value.state() == ShardState.MISSING) {
            if (metaData.hasIndex(entry.key.getIndex().getName()) &&
                metaData.getIndexSafe(entry.key.getIndex()).getState() == IndexMetaData.State.CLOSE) {
                closed.add(entry.key.getIndex().getName());
            } else {
                missing.add(entry.key.getIndex().getName());
            }
        }
    }
    return new Tuple<>(missing, closed);
}
 
Example 9
Source Project: Elasticsearch   Source File: GatewayMetaState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Loads the current meta state for each index in the new cluster state and checks if it has to be persisted.
 * Each index state that should be written to disk will be returned. This is only run for data only nodes.
 * It will return only the states for indices that actually have a shard allocated on the current node.
 *
 * @param previouslyWrittenIndices    A list of indices for which the state was already written before
 * @param potentiallyUnwrittenIndices The list of indices for which state should potentially be written
 * @param previousMetaData            The last meta data we know of. meta data for all indices in previouslyWrittenIndices list is persisted now
 * @param newMetaData                 The new metadata
 * @return iterable over all indices states that should be written to disk
 */
public static Iterable<GatewayMetaState.IndexMetaWriteInfo> resolveStatesToBeWritten(ImmutableSet<String> previouslyWrittenIndices, Set<String> potentiallyUnwrittenIndices, MetaData previousMetaData, MetaData newMetaData) {
    List<GatewayMetaState.IndexMetaWriteInfo> indicesToWrite = new ArrayList<>();
    for (String index : potentiallyUnwrittenIndices) {
        IndexMetaData newIndexMetaData = newMetaData.index(index);
        IndexMetaData previousIndexMetaData = previousMetaData == null ? null : previousMetaData.index(index);
        String writeReason = null;
        if (previouslyWrittenIndices.contains(index) == false || previousIndexMetaData == null) {
            writeReason = "freshly created";
        } else if (previousIndexMetaData.getVersion() != newIndexMetaData.getVersion()) {
            writeReason = "version changed from [" + previousIndexMetaData.getVersion() + "] to [" + newIndexMetaData.getVersion() + "]";
        }
        if (writeReason != null) {
            indicesToWrite.add(new GatewayMetaState.IndexMetaWriteInfo(newIndexMetaData, previousIndexMetaData, writeReason));
        }
    }
    return indicesToWrite;
}
 
Example 10
Source Project: crate   Source File: GatewayMetaState.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Elasticsearch 2.0 removed several deprecated features and as well as support for Lucene 3.x. This method calls
 * {@link MetaDataIndexUpgradeService} to makes sure that indices are compatible with the current version. The
 * MetaDataIndexUpgradeService might also update obsolete settings if needed.
 * Allows upgrading global custom meta data via {@link MetaDataUpgrader#customMetaDataUpgraders}
 *
 * @return input <code>metaData</code> if no upgrade is needed or an upgraded metaData
 */
static MetaData upgradeMetaData(MetaData metaData,
                                MetaDataIndexUpgradeService metaDataIndexUpgradeService,
                                MetaDataUpgrader metaDataUpgrader) {
    // upgrade index meta data
    boolean changed = false;
    final MetaData.Builder upgradedMetaData = MetaData.builder(metaData);
    for (IndexMetaData indexMetaData : metaData) {
        IndexMetaData newMetaData = metaDataIndexUpgradeService.upgradeIndexMetaData(indexMetaData,
                Version.CURRENT.minimumIndexCompatibilityVersion());
        changed |= indexMetaData != newMetaData;
        upgradedMetaData.put(newMetaData, false);
    }
    // upgrade global custom meta data
    if (applyPluginUpgraders(metaData.getCustoms(), metaDataUpgrader.customMetaDataUpgraders,
            upgradedMetaData::removeCustom, upgradedMetaData::putCustom)) {
        changed = true;
    }
    // upgrade current templates
    if (applyPluginUpgraders(metaData.getTemplates(), metaDataUpgrader.indexTemplateMetaDataUpgraders,
            upgradedMetaData::removeTemplate, (s, indexTemplateMetaData) -> upgradedMetaData.put(indexTemplateMetaData))) {
        changed = true;
    }
    return changed ? upgradedMetaData.build() : metaData;
}
 
Example 11
Source Project: crate   Source File: OperationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildFromSingleIndexBlocks() throws Exception {
    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder().put(
        IndexMetaData.SETTING_READ_ONLY, true).build(), IndexMetaData.State.OPEN),
        is(Operation.READ_ONLY));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_READ, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.UPDATE, Operation.INSERT, Operation.DELETE, Operation.DROP, Operation.ALTER,
            Operation.ALTER_OPEN_CLOSE, Operation.ALTER_BLOCKS, Operation.REFRESH, Operation.OPTIMIZE, Operation.ALTER_REROUTE));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_WRITE, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.READ, Operation.ALTER, Operation.ALTER_OPEN_CLOSE, Operation.ALTER_BLOCKS,
            Operation.SHOW_CREATE, Operation.REFRESH, Operation.OPTIMIZE, Operation.COPY_TO,
            Operation.CREATE_SNAPSHOT, Operation.ALTER_REROUTE));

    assertThat(Operation.buildFromIndexSettingsAndState(Settings.builder()
            .put(IndexMetaData.SETTING_BLOCKS_METADATA, true).build(), IndexMetaData.State.OPEN),
        containsInAnyOrder(Operation.READ, Operation.UPDATE, Operation.INSERT, Operation.DELETE, Operation.ALTER_BLOCKS,
            Operation.ALTER_OPEN_CLOSE, Operation.REFRESH, Operation.SHOW_CREATE, Operation.OPTIMIZE, Operation.ALTER_REROUTE));
}
 
Example 12
Source Project: Elasticsearch   Source File: IndexRoutingTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Initializes an index, to be restored from snapshot
 */
private Builder initializeAsRestore(IndexMetaData indexMetaData, RestoreSource restoreSource, IntSet ignoreShards, boolean asNew, UnassignedInfo unassignedInfo) {
    if (!shards.isEmpty()) {
        throw new IllegalStateException("trying to initialize an index with fresh shards, but already has shards created");
    }
    for (int shardId = 0; shardId < indexMetaData.getNumberOfShards(); shardId++) {
        IndexShardRoutingTable.Builder indexShardRoutingBuilder = new IndexShardRoutingTable.Builder(new ShardId(indexMetaData.getIndex(), shardId));
        for (int i = 0; i <= indexMetaData.getNumberOfReplicas(); i++) {
            if (asNew && ignoreShards.contains(shardId)) {
                // This shards wasn't completely snapshotted - restore it as new shard
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, null, i == 0, unassignedInfo));
            } else {
                indexShardRoutingBuilder.addShard(ShardRouting.newUnassigned(index, shardId, i == 0 ? restoreSource : null, i == 0, unassignedInfo));
            }
        }
        shards.put(shardId, indexShardRoutingBuilder.build());
    }
    return this;
}
 
Example 13
Source Project: Elasticsearch   Source File: InternalClusterInfoService.java    License: Apache License 2.0 6 votes vote down vote up
static void buildShardLevelInfo(ESLogger logger, ShardStats[] stats, HashMap<String, Long> newShardSizes, HashMap<ShardRouting, String> newShardRoutingToDataPath, ClusterState state) {
    MetaData meta = state.getMetaData();
    for (ShardStats s : stats) {
        IndexMetaData indexMeta = meta.index(s.getShardRouting().index());
        Settings indexSettings = indexMeta == null ? null : indexMeta.getSettings();
        newShardRoutingToDataPath.put(s.getShardRouting(), s.getDataPath());
        long size = s.getStats().getStore().sizeInBytes();
        String sid = ClusterInfo.shardIdentifierFromRouting(s.getShardRouting());
        if (logger.isTraceEnabled()) {
            logger.trace("shard: {} size: {}", sid, size);
        }
        if (indexSettings != null && IndexMetaData.isIndexUsingShadowReplicas(indexSettings)) {
            // Shards on a shared filesystem should be considered of size 0
            if (logger.isTraceEnabled()) {
                logger.trace("shard: {} is using shadow replicas and will be treated as size 0", sid);
            }
            size = 0;
        }
        newShardSizes.put(sid, size);
    }
}
 
Example 14
Source Project: crate   Source File: NodeRepurposeCommand.java    License: Apache License 2.0 5 votes vote down vote up
private String toIndexName(NodeEnvironment.NodePath[] nodePaths, String uuid) {
    Path[] indexPaths = new Path[nodePaths.length];
    for (int i = 0; i < nodePaths.length; i++) {
        indexPaths[i] = nodePaths[i].resolve(uuid);
    }
    try {
        IndexMetaData metaData = IndexMetaData.FORMAT.loadLatestState(LOGGER, namedXContentRegistry, indexPaths);
        return metaData.getIndex().getName();
    } catch (Exception e) {
        return "no name for uuid: " + uuid + ": " + e;
    }
}
 
Example 15
Source Project: crate   Source File: IndexSettingsModule.java    License: Apache License 2.0 5 votes vote down vote up
public static IndexSettings newIndexSettings(final IndexMetaData indexMetaData, Setting<?>... setting) {
    Set<Setting<?>> settingSet = new HashSet<>(IndexScopedSettings.BUILT_IN_INDEX_SETTINGS);
    if (setting.length > 0) {
        settingSet.addAll(Arrays.asList(setting));
    }
    return new IndexSettings(indexMetaData, Settings.EMPTY, new IndexScopedSettings(Settings.EMPTY, settingSet));
}
 
Example 16
Source Project: crate   Source File: ShardsLimitAllocationDecider.java    License: Apache License 2.0 5 votes vote down vote up
private Decision doDecide(ShardRouting shardRouting, RoutingNode node, RoutingAllocation allocation,
                          BiPredicate<Integer, Integer> decider) {
    IndexMetaData indexMd = allocation.metaData().getIndexSafe(shardRouting.index());
    final int indexShardLimit = INDEX_TOTAL_SHARDS_PER_NODE_SETTING.get(indexMd.getSettings(), settings);
    // Capture the limit here in case it changes during this method's
    // execution
    final int clusterShardLimit = this.clusterShardLimit;

    if (indexShardLimit <= 0 && clusterShardLimit <= 0) {
        return allocation.decision(Decision.YES, NAME, "total shard limits are disabled: [index: %d, cluster: %d] <= 0",
                indexShardLimit, clusterShardLimit);
    }

    int indexShardCount = 0;
    int nodeShardCount = 0;
    for (ShardRouting nodeShard : node) {
        // don't count relocating shards...
        if (nodeShard.relocating()) {
            continue;
        }
        nodeShardCount++;
        if (nodeShard.index().equals(shardRouting.index())) {
            indexShardCount++;
        }
    }

    if (clusterShardLimit > 0 && decider.test(nodeShardCount, clusterShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node, cluster setting [%s=%d]",
            nodeShardCount, CLUSTER_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), clusterShardLimit);
    }
    if (indexShardLimit > 0 && decider.test(indexShardCount, indexShardLimit)) {
        return allocation.decision(Decision.NO, NAME,
            "too many shards [%d] allocated to this node for index [%s], index setting [%s=%d]",
            indexShardCount, shardRouting.getIndexName(), INDEX_TOTAL_SHARDS_PER_NODE_SETTING.getKey(), indexShardLimit);
    }
    return allocation.decision(Decision.YES, NAME,
        "the shard count [%d] for this node is under the index limit [%d] and cluster level node limit [%d]",
        nodeShardCount, indexShardLimit, clusterShardLimit);
}
 
Example 17
Source Project: crate   Source File: DocIndexMetaDataTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSchemaWithNotNullGeneratedColumn() throws Exception {
    XContentBuilder builder = XContentFactory.jsonBuilder()
        .startObject()
            .startObject("_meta")
                .startObject("generated_columns")
                    .field("week", "date_trunc('week', ts)")
                .endObject()
                .startObject("constraints")
                    .array("not_null", "week")
                .endObject()
            .endObject()
            .startObject("properties")
                .startObject("ts").field("type", "date").endObject()
                .startObject("week").field("type", "long").endObject()
            .endObject()
        .endObject();

    IndexMetaData metaData = getIndexMetaData("test1", builder);
    DocIndexMetaData md = newMeta(metaData, "test1");

    assertThat(md.columns().size(), is(2));
    Reference week = md.references().get(new ColumnIdent("week"));
    assertThat(week, Matchers.notNullValue());
    assertThat(week.isNullable(), is(false));
    assertThat(week, instanceOf(GeneratedReference.class));
    assertThat(((GeneratedReference) week).formattedGeneratedExpression(), is("date_trunc('week', ts)"));
    assertThat(((GeneratedReference) week).generatedExpression(), isFunction("date_trunc", isLiteral("week"), isReference("ts")));
    assertThat(((GeneratedReference) week).referencedReferences(), contains(isReference("ts")));
}
 
Example 18
Source Project: crate   Source File: OperationRouting.java    License: Apache License 2.0 5 votes vote down vote up
public static int calculateScaledShardId(IndexMetaData indexMetaData, String effectiveRouting, int partitionOffset) {
    final int hash = Murmur3HashFunction.hash(effectiveRouting) + partitionOffset;

    // we don't use IMD#getNumberOfShards since the index might have been shrunk such that we need to use the size
    // of original index to hash documents
    return Math.floorMod(hash, indexMetaData.getRoutingNumShards()) / indexMetaData.getRoutingFactor();
}
 
Example 19
Source Project: crate   Source File: LocalAllocateDangledIndices.java    License: Apache License 2.0 5 votes vote down vote up
public void allocateDangled(Collection<IndexMetaData> indices, final Listener listener) {
    ClusterState clusterState = clusterService.state();
    DiscoveryNode masterNode = clusterState.nodes().getMasterNode();
    if (masterNode == null) {
        listener.onFailure(new MasterNotDiscoveredException("no master to send allocate dangled request"));
        return;
    }
    AllocateDangledRequest request = new AllocateDangledRequest(clusterService.localNode(),
        indices.toArray(new IndexMetaData[indices.size()]));
    transportService.sendRequest(masterNode, ACTION_NAME, request, new TransportResponseHandler<AllocateDangledResponse>() {
        @Override
        public AllocateDangledResponse read(StreamInput in) throws IOException {
            return new AllocateDangledResponse(in);
        }

        @Override
        public void handleResponse(AllocateDangledResponse response) {
            listener.onResponse(response);
        }

        @Override
        public void handleException(TransportException exp) {
            listener.onFailure(exp);
        }

        @Override
        public String executor() {
            return ThreadPool.Names.SAME;
        }
    });
}
 
Example 20
Source Project: Elasticsearch   Source File: DocIndexMetaData.java    License: Apache License 2.0 5 votes vote down vote up
protected DocIndexMetaData merge(DocIndexMetaData other,
                                 TransportPutIndexTemplateAction transportPutIndexTemplateAction,
                                 boolean thisIsCreatedFromTemplate) throws IOException {
    if (schemaEquals(other)) {
        return this;
    } else if (thisIsCreatedFromTemplate) {
        if (this.references.size() < other.references.size()) {
            // this is older, update template and return other
            // settings in template are always authoritative for table information about
            // number_of_shards and number_of_replicas
            updateTemplate(other, transportPutIndexTemplateAction, this.metaData.getSettings());
            // merge the new mapping with the template settings
            return new DocIndexMetaData(
                    functions,
                    IndexMetaData.builder(other.metaData).settings(this.metaData.getSettings()).build(),
                    other.ident).build();
        } else if (references().size() == other.references().size() &&
                   !references().keySet().equals(other.references().keySet())) {
            XContentHelper.update(defaultMappingMap, other.defaultMappingMap, false);
            // update the template with new information
            updateTemplate(this, transportPutIndexTemplateAction, this.metaData.getSettings());
            return this;
        }
        // other is older, just return this
        return this;
    } else {
        throw new TableAliasSchemaException(other.ident.name());
    }
}
 
Example 21
Source Project: Elasticsearch   Source File: AllocationService.java    License: Apache License 2.0 5 votes vote down vote up
public static void updateLeftDelayOfUnassignedShards(RoutingAllocation allocation, Settings settings) {
    for (ShardRouting shardRouting : allocation.routingNodes().unassigned()) {
        final MetaData metaData = allocation.metaData();
        final IndexMetaData indexMetaData = metaData.index(shardRouting.index());
        shardRouting.unassignedInfo().updateDelay(allocation.getCurrentNanoTime(), settings, indexMetaData.getSettings());
    }
}
 
Example 22
Source Project: Elasticsearch   Source File: RoutingTable.java    License: Apache License 2.0 5 votes vote down vote up
public Builder addAsFromCloseToOpen(IndexMetaData indexMetaData) {
    if (indexMetaData.getState() == IndexMetaData.State.OPEN) {
        IndexRoutingTable.Builder indexRoutingBuilder = new IndexRoutingTable.Builder(indexMetaData.getIndex())
                .initializeAsFromCloseToOpen(indexMetaData);
        add(indexRoutingBuilder);
    }
    return this;
}
 
Example 23
public static MapperService newMapperService(Settings settings, Client client) {
    Settings indexSettings = Settings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("path.home", System.getProperty("path.home"))
            .put("client.type", "node")
            .put(settings)
            .build();
    Index index = new Index("test");
    Injector parentInjector = new ModulesBuilder()
            .add(new SettingsModule(indexSettings),
            new EnvironmentModule(new Environment(indexSettings)))
            .createInjector();
    AnalysisModule analysisModule = new AnalysisModule(indexSettings,
            parentInjector.getInstance(IndicesAnalysisService.class));
    new AnalysisBaseformPlugin(settings).onModule(analysisModule);
    Injector injector = new ModulesBuilder().add(new IndexSettingsModule(index, indexSettings),
            new IndexNameModule(index),
            analysisModule)
            .createChildInjector(parentInjector);
    AnalysisService analysisService = injector.getInstance(AnalysisService.class);
    SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, indexSettings);
    Map<String, Mapper.TypeParser> mappers = registerBuiltInMappers();
    Map<String, MetadataFieldMapper.TypeParser> metadataMappers = registerBuiltInMetadataMappers();
    MapperRegistry mapperRegistry = new MapperRegistry(mappers, metadataMappers);
    return new MapperService(new Index("test"),
            indexSettings,
            analysisService,
            similarityLookupService,
            null,
            mapperRegistry);
}
 
Example 24
Source Project: crate   Source File: DocIndexMetaDataTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSchemaWithNotNullNestedColumns() throws Exception {
    XContentBuilder builder = XContentFactory.jsonBuilder()
    .startObject()
        .startObject(Constants.DEFAULT_MAPPING_TYPE)
            .startObject("_meta")
                .startObject("constraints")
                    .array("not_null", "nested.level1", "nested.level1.level2")
                .endObject()
            .endObject()
            .startObject("properties")
                .startObject("nested")
                    .field("type", "object")
                        .startObject("properties")
                            .startObject("level1")
                                .field("type", "object")
                                .startObject("properties")
                                    .startObject("level2")
                                        .field("type", "string")
                                    .endObject()
                                .endObject()
                            .endObject()
                         .endObject()
                .endObject()
            .endObject()
        .endObject()
    .endObject();
    IndexMetaData metaData = getIndexMetaData("test_notnull_columns", builder);
    DocIndexMetaData md = newMeta(metaData, "test_notnull_columns");

    ColumnIdent level1 = new ColumnIdent("nested", "level1");
    ColumnIdent level2 = new ColumnIdent("nested", Arrays.asList("level1", "level2"));
    assertThat(md.notNullColumns(), containsInAnyOrder(level1, level2));
    assertThat(md.references().get(level1).isNullable(), is(false));
    assertThat(md.references().get(level2).isNullable(), is(false));
}
 
Example 25
Source Project: Elasticsearch   Source File: IndicesTTLService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the shards to purge, i.e. the local started primary shards that have ttl enabled and disable_purge to false
 */
private List<IndexShard> getShardsToPurge() {
    List<IndexShard> shardsToPurge = new ArrayList<>();
    MetaData metaData = clusterService.state().metaData();
    for (IndexService indexService : indicesService) {
        // check the value of disable_purge for this index
        IndexMetaData indexMetaData = metaData.index(indexService.index().name());
        if (indexMetaData == null) {
            continue;
        }
        boolean disablePurge = indexMetaData.getSettings().getAsBoolean(INDEX_TTL_DISABLE_PURGE, false);
        if (disablePurge) {
            continue;
        }

        // check if ttl is enabled for at least one type of this index
        boolean hasTTLEnabled = false;
        for (String type : indexService.mapperService().types()) {
            DocumentMapper documentType = indexService.mapperService().documentMapper(type);
            if (documentType.TTLFieldMapper().enabled()) {
                hasTTLEnabled = true;
                break;
            }
        }
        if (hasTTLEnabled) {
            for (IndexShard indexShard : indexService) {
                if (indexShard.state() == IndexShardState.STARTED && indexShard.routingEntry().primary() && indexShard.routingEntry().started()) {
                    shardsToPurge.add(indexShard);
                }
            }
        }
    }
    return shardsToPurge;
}
 
Example 26
Source Project: crate   Source File: IndexService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void updateMetaData(final IndexMetaData currentIndexMetaData, final IndexMetaData newIndexMetaData) {
    final Translog.Durability oldTranslogDurability = indexSettings.getTranslogDurability();

    final boolean updateIndexMetaData = indexSettings.updateIndexMetaData(newIndexMetaData);

    if (Assertions.ENABLED
            && currentIndexMetaData != null
            && currentIndexMetaData.getCreationVersion().onOrAfter(Version.ES_V_6_5_1)) {
        final long currentSettingsVersion = currentIndexMetaData.getSettingsVersion();
        final long newSettingsVersion = newIndexMetaData.getSettingsVersion();
        if (currentSettingsVersion == newSettingsVersion) {
            assert updateIndexMetaData == false;
        } else {
            assert updateIndexMetaData;
            assert currentSettingsVersion < newSettingsVersion :
                    "expected current settings version [" + currentSettingsVersion + "] "
                            + "to be less than new settings version [" + newSettingsVersion + "]";
        }
    }

    if (updateIndexMetaData) {
        for (final IndexShard shard : this.shards.values()) {
            try {
                shard.onSettingsChanged();
            } catch (Exception e) {
                logger.warn(
                    () -> new ParameterizedMessage(
                        "[{}] failed to notify shard about setting change", shard.shardId().id()), e);
            }
        }
        if (refreshTask.getInterval().equals(indexSettings.getRefreshInterval()) == false) {
            rescheduleRefreshTasks();
        }
        final Translog.Durability durability = indexSettings.getTranslogDurability();
        if (durability != oldTranslogDurability) {
            rescheduleFsyncTask(durability);
        }
    }
}
 
Example 27
Source Project: crate   Source File: MapperService.java    License: Apache License 2.0 5 votes vote down vote up
private void assertMappingVersion(
        final IndexMetaData currentIndexMetaData,
        final IndexMetaData newIndexMetaData,
        final Map<String, DocumentMapper> updatedEntries) {
    if (Assertions.ENABLED
            && currentIndexMetaData != null
            && currentIndexMetaData.getCreationVersion().onOrAfter(Version.ES_V_6_5_1)) {
        if (currentIndexMetaData.getMappingVersion() == newIndexMetaData.getMappingVersion()) {
            // if the mapping version is unchanged, then there should not be any updates and all mappings should be the same
            assert updatedEntries.isEmpty() : updatedEntries;
            for (final ObjectCursor<MappingMetaData> mapping : newIndexMetaData.getMappings().values()) {
                final CompressedXContent currentSource = currentIndexMetaData.mapping(mapping.value.type()).source();
                final CompressedXContent newSource = mapping.value.source();
                assert currentSource.equals(newSource) :
                        "expected current mapping [" + currentSource + "] for type [" + mapping.value.type() + "] "
                                + "to be the same as new mapping [" + newSource + "]";
            }
        } else {
            // if the mapping version is changed, it should increase, there should be updates, and the mapping should be different
            final long currentMappingVersion = currentIndexMetaData.getMappingVersion();
            final long newMappingVersion = newIndexMetaData.getMappingVersion();
            assert currentMappingVersion < newMappingVersion :
                    "expected current mapping version [" + currentMappingVersion + "] "
                            + "to be less than new mapping version [" + newMappingVersion + "]";
            assert updatedEntries.isEmpty() == false;
            for (final DocumentMapper documentMapper : updatedEntries.values()) {
                final MappingMetaData currentMapping = currentIndexMetaData.mapping(documentMapper.type());
                if (currentMapping != null) {
                    final CompressedXContent currentSource = currentMapping.source();
                    final CompressedXContent newSource = documentMapper.mappingSource();
                    assert currentSource.equals(newSource) == false :
                            "expected current mapping [" + currentSource + "] for type [" + documentMapper.type() + "] " +
                                    "to be different than new mapping";
                }
            }
        }
    }
}
 
Example 28
Source Project: crate   Source File: TablesNeedUpgradeSysCheck.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<?> computeResult() {
    HashSet<String> fqTables = new HashSet<>();
    for (ObjectCursor<IndexMetaData> cursor : clusterService.state()
        .metaData()
        .indices()
        .values()) {

        if (cursor.value.getCreationVersion().major < org.elasticsearch.Version.CURRENT.major) {
            fqTables.add(RelationName.fqnFromIndexName(cursor.value.getIndex().getName()));
        }
    }
    tablesNeedUpgrade = fqTables;
    return CompletableFuture.completedFuture(fqTables);
}
 
Example 29
Source Project: crate   Source File: AlterTableOperationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNumberOfShardsRequestedNotAFactorOfCurrentIsNotSupported() {
    IndexMetaData indexMetaData = IndexMetaData.builder("t1")
        .settings(baseIndexSettings())
        .build();

    expectedException.expect(IllegalArgumentException.class);
    expectedException.expectMessage("Requested number of shards: <3> needs to be a factor of the current one: <5>");
    AlterTableOperation.validateNumberOfShardsForResize(indexMetaData, 3);
}
 
Example 30
Source Project: Elasticsearch   Source File: Version.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Return the {@link Version} of Elasticsearch that has been used to create an index given its settings.
 *
 * @throws IllegalStateException if the given index settings doesn't contain a value for the key {@value IndexMetaData#SETTING_VERSION_CREATED}
 */
public static Version indexCreated(Settings indexSettings) {
    final Version indexVersion = indexSettings.getAsVersion(IndexMetaData.SETTING_VERSION_CREATED, null);
    if (indexVersion == null) {
        throw new IllegalStateException("[" + IndexMetaData.SETTING_VERSION_CREATED + "] is not present in the index settings for index with uuid: [" + indexSettings.get(IndexMetaData.SETTING_INDEX_UUID) + "]");
    }
    return indexVersion;
}