org.elasticsearch.discovery.DiscoveryModule Java Examples

public HttpHost start() {
    var watch = new StopWatch();
    this.dataPath = Files.tempDir();
    try {
        Settings.Builder settings = Settings.builder();
        settings.put(ClusterName.CLUSTER_NAME_SETTING.getKey(), "test")
                .put(Node.NODE_NAME_SETTING.getKey(), "test")
                .put(Environment.PATH_HOME_SETTING.getKey(), dataPath)
                .put(NetworkService.GLOBAL_NETWORK_BIND_HOST_SETTING.getKey(), "_local_")
                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE);
        node = new LocalNode(settings.build());
        node.start();
        // on same local server, there may be multiple es started, e.g. multiple test jobs on shared build server, this is to retrieve actual http port
        return new HttpHost("localhost", node.injector().getInstance(HttpServerTransport.class).boundAddress().publishAddress().getPort());
    } catch (NodeValidationException e) {
        throw new Error(e);
    } finally {
        logger.info("create local elasticsearch node, dataPath={}, elapsed={}", dataPath, watch.elapsed());
    }
}
 

@Override
protected void doStart() {
    synchronized (mutex) {
        CoordinationState.PersistedState persistedState = persistedStateSupplier.get();
        coordinationState.set(new CoordinationState(settings, getLocalNode(), persistedState));
        peerFinder.setCurrentTerm(getCurrentTerm());
        configuredHostsResolver.start();
        VotingConfiguration votingConfiguration = coordinationState.get().getLastAcceptedState().getLastCommittedConfiguration();
        if (singleNodeDiscovery &&
            votingConfiguration.isEmpty() == false &&
            votingConfiguration.hasQuorum(Collections.singleton(getLocalNode().getId())) == false) {
            throw new IllegalStateException("cannot start with [" + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() + "] set to [" +
                DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE + "] when local node " + getLocalNode() +
                " does not have quorum in voting configuration " + votingConfiguration);
        }
        ClusterState initialState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.get(settings))
            .blocks(ClusterBlocks.builder()
                .addGlobalBlock(STATE_NOT_RECOVERED_BLOCK)
                .addGlobalBlock(noMasterBlockService.getNoMasterBlock()))
            .nodes(DiscoveryNodes.builder().add(getLocalNode()).localNodeId(getLocalNode().getId()))
            .build();
        applierState = initialState;
        clusterApplier.setInitialState(initialState);
    }
}
 

public void testBootstrapsAutomaticallyWithSingleNodeDiscovery() {
    final Settings.Builder settings = Settings.builder()
        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE)
        .put(NODE_NAME_SETTING.getKey(), localNode.getName());
    final AtomicBoolean bootstrapped = new AtomicBoolean();

    ClusterBootstrapService clusterBootstrapService = new ClusterBootstrapService(settings.build(),
        transportService, () -> emptyList(), () -> false, vc -> {
        assertTrue(bootstrapped.compareAndSet(false, true));
        assertThat(vc.getNodeIds(), hasSize(1));
        assertThat(vc.getNodeIds(), hasItem(localNode.getId()));
        assertTrue(vc.hasQuorum(singletonList(localNode.getId())));
    });

    transportService.start();
    clusterBootstrapService.onFoundPeersUpdated();
    deterministicTaskQueue.runAllTasks();
    assertTrue(bootstrapped.get());

    bootstrapped.set(false);
    clusterBootstrapService.onFoundPeersUpdated();
    deterministicTaskQueue.runAllTasks();
    assertFalse(bootstrapped.get()); // should only bootstrap once
}
 

public void onModule(DiscoveryModule discoveryModule) {
    /**
     * Different types of discovery modules can be defined on startup using the `discovery.type` setting.
     * This SrvDiscoveryModule can be loaded using `-Des.discovery.type=srv`
     */
    if ("srv".equals(settings.get("discovery.type"))) {
        discoveryModule.addDiscoveryType(SrvDiscovery.SRV, SrvDiscovery.class);
        discoveryModule.addUnicastHostProvider(SrvUnicastHostsProvider.class);
    }
}
 

public Coordinator(String nodeName, Settings settings, ClusterSettings clusterSettings, TransportService transportService,
                   NamedWriteableRegistry namedWriteableRegistry, AllocationService allocationService, MasterService masterService,
                   Supplier<CoordinationState.PersistedState> persistedStateSupplier, SeedHostsProvider seedHostsProvider,
                   ClusterApplier clusterApplier, Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators, Random random) {
    this.settings = settings;
    this.transportService = transportService;
    this.masterService = masterService;
    this.allocationService = allocationService;
    this.onJoinValidators = JoinTaskExecutor.addBuiltInJoinValidators(onJoinValidators);
    this.singleNodeDiscovery = DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings));
    this.joinHelper = new JoinHelper(settings, allocationService, masterService, transportService,
        this::getCurrentTerm, this::getStateForMasterService, this::handleJoinRequest, this::joinLeaderInTerm, this.onJoinValidators);
    this.persistedStateSupplier = persistedStateSupplier;
    this.noMasterBlockService = new NoMasterBlockService(settings, clusterSettings);
    this.lastKnownLeader = Optional.empty();
    this.lastJoin = Optional.empty();
    this.joinAccumulator = new InitialJoinAccumulator();
    this.publishTimeout = PUBLISH_TIMEOUT_SETTING.get(settings);
    this.random = random;
    this.electionSchedulerFactory = new ElectionSchedulerFactory(settings, random, transportService.getThreadPool());
    this.preVoteCollector = new PreVoteCollector(transportService, this::startElection, this::updateMaxTermSeen);
    configuredHostsResolver = new SeedHostsResolver(nodeName, settings, transportService, seedHostsProvider);
    this.peerFinder = new CoordinatorPeerFinder(settings, transportService,
        new HandshakingTransportAddressConnector(settings, transportService), configuredHostsResolver);
    this.publicationHandler = new PublicationTransportHandler(transportService, namedWriteableRegistry,
        this::handlePublishRequest, this::handleApplyCommit);
    this.leaderChecker = new LeaderChecker(settings, transportService, getOnLeaderFailure());
    this.followersChecker = new FollowersChecker(settings, transportService, this::onFollowerCheckRequest, this::removeNode);
    this.nodeRemovalExecutor = new NodeRemovalClusterStateTaskExecutor(allocationService, LOGGER);
    this.clusterApplier = clusterApplier;
    masterService.setClusterStateSupplier(this::getStateForMasterService);
    this.reconfigurator = new Reconfigurator(settings, clusterSettings);
    this.clusterBootstrapService = new ClusterBootstrapService(settings, transportService, this::getFoundPeers,
        this::isInitialConfigurationSet, this::setInitialConfiguration);
    this.lagDetector = new LagDetector(settings, transportService.getThreadPool(), n -> removeNode(n, "lagging"),
        transportService::getLocalNode);
    this.clusterFormationFailureHelper = new ClusterFormationFailureHelper(settings, this::getClusterFormationState,
        transportService.getThreadPool(), joinHelper::logLastFailedJoinAttempt);
}
 

public ClusterBootstrapService(Settings settings, TransportService transportService,
                               Supplier<Iterable<DiscoveryNode>> discoveredNodesSupplier, BooleanSupplier isBootstrappedSupplier,
                               Consumer<VotingConfiguration> votingConfigurationConsumer) {
    if (DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE.equals(DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings))) {
        if (INITIAL_MASTER_NODES_SETTING.exists(settings)) {
            throw new IllegalArgumentException("setting [" + INITIAL_MASTER_NODES_SETTING.getKey() +
                "] is not allowed when [" + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() + "] is set to [" +
                DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE + "]");
        }
        if (DiscoveryNode.isMasterNode(settings) == false) {
            throw new IllegalArgumentException("node with [" + DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey() + "] set to [" +
                DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE + "] must be master-eligible");
        }
        bootstrapRequirements = Collections.singleton(Node.NODE_NAME_SETTING.get(settings));
        unconfiguredBootstrapTimeout = null;
    } else {
        final List<String> initialMasterNodes = INITIAL_MASTER_NODES_SETTING.get(settings);
        bootstrapRequirements = unmodifiableSet(new LinkedHashSet<>(initialMasterNodes));
        if (bootstrapRequirements.size() != initialMasterNodes.size()) {
            throw new IllegalArgumentException(
                "setting [" + INITIAL_MASTER_NODES_SETTING.getKey() + "] contains duplicates: " + initialMasterNodes);
        }
        unconfiguredBootstrapTimeout = discoveryIsConfigured(settings) ? null : UNCONFIGURED_BOOTSTRAP_TIMEOUT_SETTING.get(settings);
    }

    this.transportService = transportService;
    this.discoveredNodesSupplier = discoveredNodesSupplier;
    this.isBootstrappedSupplier = isBootstrappedSupplier;
    this.votingConfigurationConsumer = votingConfigurationConsumer;
}
 

/**
 * Executes the bootstrap checks if the node has the transport protocol bound to a non-loopback interface. If the system property
 * {@code es.enforce.bootstrap.checks} is set to {@code true} then the bootstrap checks will be enforced regardless of whether or not
 * the transport protocol is bound to a non-loopback interface.
 *
 * @param boundTransportAddress the node network bindings
 */
static void check(final Settings settings,
                  final BoundTransportAddress boundTransportAddress,
                  List<BootstrapCheck> additionalChecks) throws NodeValidationException {
    final List<BootstrapCheck> builtInChecks = checks();
    final List<BootstrapCheck> combinedChecks = new ArrayList<>(builtInChecks);
    combinedChecks.addAll(additionalChecks);
    check(settings,
          enforceLimits(boundTransportAddress, DiscoveryModule.DISCOVERY_TYPE_SETTING.get(settings)),
          Collections.unmodifiableList(combinedChecks));
}
 

public void testSingleNodeDiscoveryWithoutQuorum() {
    final Cluster cluster = new Cluster(3);
    cluster.runRandomly();
    cluster.stabilise();

    final ClusterNode clusterNode = cluster.getAnyNode();
    logger.debug("rebooting [{}]", clusterNode.getId());
    clusterNode.close();
    cluster.clusterNodes.forEach(
        cn -> cluster.deterministicTaskQueue.scheduleNow(cn.onNode(
            new Runnable() {
                @Override
                public void run() {
                    cn.transportService.disconnectFromNode(clusterNode.getLocalNode());
                }

                @Override
                public String toString() {
                    return "disconnect from " + clusterNode.getLocalNode() + " after shutdown";
                }
            })));
    IllegalStateException ise = expectThrows(IllegalStateException.class,
        () -> cluster.clusterNodes.replaceAll(cn -> cn == clusterNode ?
            cn.restartedNode(Function.identity(), Function.identity(), Settings.builder()
                .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE).build()) :
            cn));
    assertThat(ise.getMessage(), containsString("cannot start with [discovery.type] set to [single-node] when local node"));
    assertThat(ise.getMessage(), containsString("does not have quorum in voting configuration"));
}
 

public void testFailBootstrapWithBothSingleNodeDiscoveryAndInitialMasterNodes() {
    final Settings.Builder settings = Settings.builder()
        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE)
        .put(NODE_NAME_SETTING.getKey(), localNode.getName())
        .put(INITIAL_MASTER_NODES_SETTING.getKey(), "test");

    assertThat(expectThrows(IllegalArgumentException.class, () -> new ClusterBootstrapService(settings.build(),
        transportService, () -> emptyList(), () -> false, vc -> fail())).getMessage(),
        containsString("setting [" + INITIAL_MASTER_NODES_SETTING.getKey() + "] is not allowed when [discovery.type] is set " +
            "to [single-node]"));
}
 

public void testFailBootstrapNonMasterEligibleNodeWithSingleNodeDiscovery() {
    final Settings.Builder settings = Settings.builder()
        .put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(), DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE)
        .put(NODE_NAME_SETTING.getKey(), localNode.getName())
        .put(Node.NODE_MASTER_SETTING.getKey(), false);

    assertThat(expectThrows(IllegalArgumentException.class, () -> new ClusterBootstrapService(settings.build(),
            transportService, () -> emptyList(), () -> false, vc -> fail())).getMessage(),
        containsString("node with [discovery.type] set to [single-node] must be master-eligible"));
}
 

@Override
protected Settings nodeSettings(int nodeOrdinal) {
    Settings.Builder builder = Settings.builder()
        .put(super.nodeSettings(nodeOrdinal))
        .put(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), "azure")
        .put(Management.SUBSCRIPTION_ID.getKey(), "fake")
        .put(Discovery.REFRESH.getKey(), "5s")
        .put(Management.APP_ID.getKey(), "dummy")
        .put(Management.TENANT_ID.getKey(), "dummy")
        .put(Management.APP_SECRET.getKey(), "dummy")
        .put(Management.RESOURCE_GROUP_NAME.getKey(), "dummy");
    return builder.build();
}
 

public void testSingleNodeDiscoveryWithQuorum() {
    final Cluster cluster = new Cluster(1, randomBoolean(), Settings.builder().put(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey(),
        DiscoveryModule.SINGLE_NODE_DISCOVERY_TYPE).build());
    cluster.runRandomly();
    cluster.stabilise();
}
 

private void setUp() {
    mockTransport = new DisruptableMockTransport(localNode, logger) {
        @Override
        protected void execute(Runnable runnable) {
            deterministicTaskQueue.scheduleNow(onNode(runnable));
        }

        @Override
        protected ConnectionStatus getConnectionStatus(DiscoveryNode destination) {
            return Cluster.this.getConnectionStatus(getLocalNode(), destination);
        }

        @Override
        protected Optional<DisruptableMockTransport> getDisruptableMockTransport(TransportAddress address) {
            return clusterNodes.stream().map(cn -> cn.mockTransport)
                .filter(transport -> transport.getLocalNode().getAddress().equals(address)).findAny();
        }
    };

    final Settings settings = nodeSettings.hasValue(DiscoveryModule.DISCOVERY_TYPE_SETTING.getKey()) ?
        nodeSettings : Settings.builder().put(nodeSettings)
        .putList(ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.getKey(),
            ClusterBootstrapService.INITIAL_MASTER_NODES_SETTING.get(Settings.EMPTY)).build(); // suppress auto-bootstrap
    transportService = mockTransport.createTransportService(
        settings,
        deterministicTaskQueue.getThreadPool(this::onNode),
        NOOP_TRANSPORT_INTERCEPTOR,
        a -> localNode,
        null
    );
    masterService = new AckedFakeThreadPoolMasterService(localNode.getId(), "test",
        runnable -> deterministicTaskQueue.scheduleNow(onNode(runnable)));
    final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
    clusterApplierService = new DisruptableClusterApplierService(localNode.getId(), settings, clusterSettings,
        deterministicTaskQueue, this::onNode);
    clusterService = new ClusterService(settings, clusterSettings, masterService, clusterApplierService);
    clusterService.setNodeConnectionsService(
        new NodeConnectionsService(clusterService.getSettings(), deterministicTaskQueue.getThreadPool(this::onNode),
            transportService));
    final Collection<BiConsumer<DiscoveryNode, ClusterState>> onJoinValidators =
        Collections.singletonList((dn, cs) -> extraJoinValidators.forEach(validator -> validator.accept(dn, cs)));
    coordinator = new Coordinator("test_node", settings, clusterSettings, transportService, writableRegistry(),
        ESAllocationTestCase.createAllocationService(Settings.EMPTY, clusterSettings, random()), masterService, this::getPersistedState,
        Cluster.this::provideSeedHosts, clusterApplierService, onJoinValidators, Randomness.get());
    masterService.setClusterStatePublisher(coordinator);

    logger.trace("starting up [{}]", localNode);
    transportService.start();
    transportService.acceptIncomingRequests();
    coordinator.start();
    clusterService.start();
    coordinator.startInitialJoin();
}
 

/**
 * Check if discovery is meant to start
 *
 * @param settings settings to extract cloud enabled parameter from
 * @return true if we can start discovery features
 */
public static boolean isDiscoveryReady(Settings settings, Logger logger) {
    // Cloud services are disabled
    if (!isCloudReady(settings)) {
        logger.trace("cloud settings are disabled");
        return false;
    }

    // User set discovery.zen.hosts_provider: azure
    if (!AZURE.equalsIgnoreCase(settings.get(DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey()))) {
        logger.trace("{} not set to {}", DiscoveryModule.DISCOVERY_SEED_PROVIDERS_SETTING.getKey(), AZURE);
        return false;
    }

    if (isPropertyMissing(settings, Management.SUBSCRIPTION_ID.getKey()) ||
        isPropertyMissing(settings, Management.RESOURCE_GROUP_NAME.getKey()) ||
        isPropertyMissing(settings, Management.TENANT_ID.getKey()) ||
        isPropertyMissing(settings, Management.APP_ID.getKey()) ||
        isPropertyMissing(settings, Management.APP_SECRET.getKey())
        ) {
        logger.warn("one or more azure discovery settings are missing. " +
                    "Check crate.yml file. Should have [{}], [{}], [{}] and [{}].",
            Management.SUBSCRIPTION_ID,
            Management.RESOURCE_GROUP_NAME,
            Management.TENANT_ID,
            Management.APP_ID,
            Management.APP_SECRET);
        return false;
    }

    String discoveryType = AzureComputeService.Discovery.DISCOVERY_METHOD.get(settings);
    if (!(SUBNET.equalsIgnoreCase(discoveryType) ||
          VNET.equalsIgnoreCase(discoveryType) ||
          discoveryType == null)) {
        logger.warn("{} must be set to {} or {}. Ignoring value {}", AzureComputeService.Discovery.DISCOVERY_METHOD.getKey(), VNET, SUBNET, discoveryType);
    }

    logger.trace("all required properties for azure discovery are set!");

    return true;
}