Java Code Examples for org.elasticsearch.common.settings.ClusterSettings#BUILT_IN_CLUSTER_SETTINGS

@Test
public void testDuplicateRecoveries() throws IOException {
    IndexShard primary = newStartedShard(true);
    PeerRecoverySourceService peerRecoverySourceService = new PeerRecoverySourceService(
        mock(TransportService.class), mock(IndicesService.class),
        new RecoverySettings(
            Settings.EMPTY,
            new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)));
    StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(
        primary.shardId(),
        randomAlphaOfLength(10),
        getFakeDiscoNode("source"),
        getFakeDiscoNode("target"),
        Store.MetadataSnapshot.EMPTY,
        randomBoolean(),
        randomLong(),
        SequenceNumbers.UNASSIGNED_SEQ_NO);
    RecoverySourceHandler handler = peerRecoverySourceService.ongoingRecoveries
        .addNewRecovery(startRecoveryRequest, primary);
    DelayRecoveryException delayRecoveryException = expectThrows(
        DelayRecoveryException.class,
        () -> peerRecoverySourceService.ongoingRecoveries.addNewRecovery(
            startRecoveryRequest,
            primary));
    assertThat(delayRecoveryException.getMessage(), containsString("recovery with same target already registered"));
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    // re-adding after removing previous attempt works
    handler = peerRecoverySourceService.ongoingRecoveries.addNewRecovery(startRecoveryRequest, primary);
    peerRecoverySourceService.ongoingRecoveries.remove(primary, handler);
    closeShards(primary);
}
 

public void testDynamicSetting() {
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    final Reconfigurator reconfigurator = new Reconfigurator(Settings.EMPTY, clusterSettings);
    final VotingConfiguration initialConfig = conf("a", "b", "c", "d", "e");

    Set<DiscoveryNode> twoNodes = nodes("a", "b");
    Set<DiscoveryNode> threeNodes = nodes("a", "b", "c");

    // default is "true"
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c")));

    // update to "false"
    clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "false").build());
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig),
        sameInstance(initialConfig)); // no quorum
    assertThat(reconfigurator.reconfigure(threeNodes, retired(), randomFrom(threeNodes), initialConfig),
        equalTo(conf("a", "b", "c", "d", "e")));
    assertThat(reconfigurator.reconfigure(threeNodes, retired("d"), randomFrom(threeNodes), initialConfig),
        equalTo(conf("a", "b", "c", "e")));

    // explicitly set to "true"
    clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "true").build());
    assertThat(reconfigurator.reconfigure(twoNodes, retired(), randomFrom(twoNodes), initialConfig), equalTo(conf("a", "b", "c")));

    expectThrows(IllegalArgumentException.class, () ->
        clusterSettings.applySettings(Settings.builder().put(CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION.getKey(), "blah").build()));
}
 

public BalancedShardsAllocator(Settings settings) {
    this(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
 

/**
 * creates a new initializing shard.
 * @param routing                       shard routing to use
 * @param shardPath                     path to use for shard data
 * @param indexMetaData                 indexMetaData for the shard, including any mapping
 * @param storeProvider                 an optional custom store provider to use. If null a default file based store will be created
 * @param indexSearcherWrapper          an optional wrapper to be used during searchers
 * @param globalCheckpointSyncer        callback for syncing global checkpoints
 * @param indexEventListener            index event listener
 * @param listeners                     an optional set of listeners to add to the shard
 */
protected IndexShard newShard(ShardRouting routing, ShardPath shardPath, IndexMetaData indexMetaData,
                              @Nullable CheckedFunction<IndexSettings, Store, IOException> storeProvider,
                              @Nullable IndexSearcherWrapper indexSearcherWrapper,
                              @Nullable EngineFactory engineFactory,
                              Runnable globalCheckpointSyncer,
                              IndexEventListener indexEventListener, IndexingOperationListener... listeners) throws IOException {
    final Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
    final IndexSettings indexSettings = new IndexSettings(indexMetaData, nodeSettings);
    final IndexShard indexShard;
    if (storeProvider == null) {
        storeProvider = is -> createStore(is, shardPath);
    }
    final Store store = storeProvider.apply(indexSettings);
    boolean success = false;
    try {
        IndexCache indexCache = new IndexCache(indexSettings, new DisabledQueryCache(indexSettings));
        MapperService mapperService = MapperTestUtils.newMapperService(xContentRegistry(), createTempDir(),
                indexSettings.getSettings(), "index");
        mapperService.merge(indexMetaData, MapperService.MergeReason.MAPPING_RECOVERY, true);
        ClusterSettings clusterSettings = new ClusterSettings(nodeSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
        CircuitBreakerService breakerService = new HierarchyCircuitBreakerService(nodeSettings, clusterSettings);
        indexShard = new IndexShard(
            routing,
            indexSettings,
            shardPath,
            store,
            indexCache,
            mapperService,
            engineFactory,
            indexEventListener,
            indexSearcherWrapper,
            threadPool,
            BigArrays.NON_RECYCLING_INSTANCE,
            Arrays.asList(listeners),
            globalCheckpointSyncer,
            breakerService
        );
        indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER);
        success = true;
    } finally {
        if (success == false) {
            IOUtils.close(store);
        }
    }
    return indexShard;
}
 

private void setupMasterServiceAndCoordinator(long term, ClusterState initialState, MasterService masterService,
                                              ThreadPool threadPool, Random random) {
    if (this.masterService != null || coordinator != null) {
        throw new IllegalStateException("method setupMasterServiceAndCoordinator can only be called once");
    }
    this.masterService = masterService;
    CapturingTransport capturingTransport = new CapturingTransport() {
        @Override
        protected void onSendRequest(long requestId, String action, TransportRequest request, DiscoveryNode destination) {
            if (action.equals(HANDSHAKE_ACTION_NAME)) {
                handleResponse(requestId, new TransportService.HandshakeResponse(destination, initialState.getClusterName(),
                    destination.getVersion()));
            } else if (action.equals(JoinHelper.VALIDATE_JOIN_ACTION_NAME)) {
                handleResponse(requestId, new TransportResponse.Empty());
            } else {
                super.onSendRequest(requestId, action, request, destination);
            }
        }
    };
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    TransportService transportService = capturingTransport.createTransportService(
        Settings.EMPTY,
        threadPool,
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> initialState.nodes().getLocalNode(),
        clusterSettings
    );
    coordinator = new Coordinator("test_node", Settings.EMPTY, clusterSettings,
        transportService, writableRegistry(),
        ESAllocationTestCase.createAllocationService(Settings.EMPTY),
        masterService,
        () -> new InMemoryPersistedState(term, initialState), r -> emptyList(),
        new NoOpClusterApplier(),
        Collections.emptyList(),
        random);
    transportService.start();
    transportService.acceptIncomingRequests();
    transport = capturingTransport;
    coordinator.start();
    coordinator.startInitialJoin();
}
 

public void testDiffSerializationFailure() {
    DeterministicTaskQueue deterministicTaskQueue =
        new DeterministicTaskQueue(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test").build(), random());
    final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    final DiscoveryNode localNode = new DiscoveryNode("localNode", buildNewFakeTransportAddress(), Version.CURRENT);
    final TransportService transportService = new CapturingTransport().createTransportService(
        Settings.EMPTY,
        deterministicTaskQueue.getThreadPool(),
        TransportService.NOOP_TRANSPORT_INTERCEPTOR,
        x -> localNode,
        clusterSettings
    );
    final PublicationTransportHandler handler = new PublicationTransportHandler(transportService,
        writableRegistry(), pu -> null, (pu, l) -> {});
    transportService.start();
    transportService.acceptIncomingRequests();

    final DiscoveryNode otherNode = new DiscoveryNode("otherNode", buildNewFakeTransportAddress(), Version.CURRENT);
    final ClusterState clusterState = CoordinationStateTests.clusterState(2L, 1L,
        DiscoveryNodes.builder().add(localNode).add(otherNode).localNodeId(localNode.getId()).build(),
        VotingConfiguration.EMPTY_CONFIG, VotingConfiguration.EMPTY_CONFIG, 0L);

    final ClusterState unserializableClusterState = new ClusterState(clusterState.version(),
        clusterState.stateUUID(), clusterState) {
        @Override
        public Diff<ClusterState> diff(ClusterState previousState) {
            return new Diff<ClusterState>() {
                @Override
                public ClusterState apply(ClusterState part) {
                    fail("this diff shouldn't be applied");
                    return part;
                }

                @Override
                public void writeTo(StreamOutput out) throws IOException {
                    throw new IOException("Simulated failure of diff serialization");
                }
            };
        }
    };

    ElasticsearchException e = expectThrows(ElasticsearchException.class, () ->
        handler.newPublicationContext(new ClusterChangedEvent("test", unserializableClusterState, clusterState)));
    assertNotNull(e.getCause());
    assertThat(e.getCause(), instanceOf(IOException.class));
    assertThat(e.getCause().getMessage(), containsString("Simulated failure of diff serialization"));
}
 

private Reconfigurator makeReconfigurator(Settings settings) {
    return new Reconfigurator(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}
 

private void setUp() {
    mockTransport = new DisruptableMockTransport(localNode, logger) {
        @Override
        protected void execute(Runnable runnable) {
            deterministicTaskQueue.scheduleNow(onNode(runnable));
        }

        @Override
        protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
            return Cluster.this.getConnectionStatus(getLocalNode(), destination);
        }

        @Override
        protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
            return clusterNodes.stream().map(cn -> cn.mockTransport)
                .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
        }
    };

    final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
        nodeSettings : Settings.builder().put(nodeSettings)
        .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
            ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
    transportService = mockTransport.createTransportService(
        settings,
        deterministicTaskQueue.getThreadPool(this::onNode),
        NOOP_TRANSPORT_INTERCEPTOR,
        a -> localNode,
        null
    );
    masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
        runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
    final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
        deterministicTaskQueue, this::onNode);
    clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
    clusterService.setNodeConnectionsService(
        new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
            transportService));
    final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
        Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
    coordinator = new Coordinator("test_node", settings, clusterSettings, transportService, writableRegistry(),
        ESAllocationTestCase.createAllocationService(Settings.EMPTY, clusterSettings, random()), masterService, this::getPersistedState,
        Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get());
    masterService.setClusterStatePublisher(coordinator);

    logger.trace("starting up [{}]", localNode);
    transportService.start();
    transportService.acceptIncomingRequests();
    coordinator.start();
    clusterService.start();
    coordinator.startInitialJoin();
}
 

@Test
public void testCanAllocateUsesMaxAvailableSpace() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);

    MetaData metaData = MetaData.builder()
        .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
        .build();

    final Index index = metaData.index("test").getIndex();

    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE,
                                                     new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(),
                                             new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(),
                                             new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);

    RoutingTable routingTable = RoutingTable.builder()
        .addAsNew(metaData.index("test"))
        .build();

    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
        .metaData(metaData).routingTable(routingTable).build();

    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                                                                .add(node_0)
                                                                .add(node_1)
    ).build();

    // actual test -- after all that bloat :)
    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0)); // all full
    leastAvailableUsages.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, 0)); // all full

    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    // 20 - 99 percent since after allocation there must be at least 10% left and shard is 10byte
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, randomIntBetween(20, 100)));
    // this is weird and smells like a bug! it should be up to 20%?
    mostAvailableUsage.put("node_1", new DiskUsage("node_1", "node_1", "_na_", 100, randomIntBetween(0, 10)));

    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    shardSizes.put("[test][0][p]", 10L); // 10 bytes
    final ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(),
                                                    mostAvailableUsage.build(), shardSizes.build(), ImmutableOpenMap.of());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
                                                         clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
    allocation.debugDecision(true);
    Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(mostAvailableUsage.toString(), Decision.Type.YES, decision.type());
    assertThat(decision.getExplanation(), containsString("enough disk for shard on node"));
    decision = decider.canAllocate(test_0, new RoutingNode("node_1", node_1), allocation);
    assertEquals(mostAvailableUsage.toString(), Decision.Type.NO, decision.type());
    assertThat(decision.getExplanation(), containsString("the node is above the high watermark cluster " +
                                                         "setting [cluster.routing.allocation.disk.watermark.high=90%], using more disk space than the maximum allowed [90.0%]"));
}
 

@Test
public void testCannotAllocateDueToLackOfDiskResources() {
    ClusterSettings nss = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    DiskThresholdDecider decider = new DiskThresholdDecider(Settings.EMPTY, nss);

    MetaData metaData = MetaData.builder()
        .put(IndexMetaData.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(1))
        .build();

    final Index index = metaData.index("test").getIndex();

    ShardRouting test_0 = ShardRouting.newUnassigned(new ShardId(index, 0), true, EmptyStoreRecoverySource.INSTANCE,
                                                     new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "foo"));
    DiscoveryNode node_0 = new DiscoveryNode("node_0", buildNewFakeTransportAddress(), Collections.emptyMap(),
                                             new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);
    DiscoveryNode node_1 = new DiscoveryNode("node_1", buildNewFakeTransportAddress(), Collections.emptyMap(),
                                             new HashSet<>(Arrays.asList(DiscoveryNode.Role.values())), Version.CURRENT);

    RoutingTable routingTable = RoutingTable.builder()
        .addAsNew(metaData.index("test"))
        .build();

    ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
        .metaData(metaData).routingTable(routingTable).build();

    clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder()
                                                                .add(node_0)
                                                                .add(node_1)
    ).build();

    // actual test -- after all that bloat :)

    ImmutableOpenMap.Builder<String, DiskUsage> leastAvailableUsages = ImmutableOpenMap.builder();
    leastAvailableUsages.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, 0)); // all full
    ImmutableOpenMap.Builder<String, DiskUsage> mostAvailableUsage = ImmutableOpenMap.builder();
    final int freeBytes = randomIntBetween(20, 100);
    mostAvailableUsage.put("node_0", new DiskUsage("node_0", "node_0", "_na_", 100, freeBytes));

    ImmutableOpenMap.Builder<String, Long> shardSizes = ImmutableOpenMap.builder();
    // way bigger than available space
    final long shardSize = randomIntBetween(110, 1000);
    shardSizes.put("[test][0][p]", shardSize);
    ClusterInfo clusterInfo = new ClusterInfo(leastAvailableUsages.build(), mostAvailableUsage.build(),
                                              shardSizes.build(), ImmutableOpenMap.of());
    RoutingAllocation allocation = new RoutingAllocation(new AllocationDeciders(Collections.singleton(decider)),
                                                         clusterState.getRoutingNodes(), clusterState, clusterInfo, System.nanoTime());
    allocation.debugDecision(true);
    Decision decision = decider.canAllocate(test_0, new RoutingNode("node_0", node_0), allocation);
    assertEquals(Decision.Type.NO, decision.type());

    assertThat(decision.getExplanation(), containsString(
        "allocating the shard to this node will bring the node above the high watermark cluster setting "
        +"[cluster.routing.allocation.disk.watermark.high=90%] "
        + "and cause it to have less than the minimum required [0b] of free space "
        + "(free: [" + freeBytes + "b], estimated shard size: [" + shardSize + "b])"));
}
 

private DiskThresholdDecider makeDecider(Settings settings) {
    return new DiskThresholdDecider(settings, new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS));
}