Java Code Examples for org.apache.solr.client.solrj.impl.CloudSolrClient#connect()

@Override
public Boolean handle(CloudSolrClient solrClient, SolrPropsConfig solrPropsConfig) throws Exception {
  boolean reloadCollectionNeeded = false;
  String separator = FileSystems.getDefault().getSeparator();
  solrClient.connect();
  SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
  try {
    ZkConfigManager zkConfigManager = new ZkConfigManager(zkClient);
    boolean configExists = zkConfigManager.configExists(solrPropsConfig.getConfigName());
    if (configExists) {
      uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
      reloadCollectionNeeded = doIfConfigExists(solrPropsConfig, zkClient, separator);
    } else {
      doIfConfigNotExist(solrPropsConfig, zkConfigManager);
      uploadMissingConfigFiles(zkClient, zkConfigManager, solrPropsConfig.getConfigName());
    }
  } catch (Exception e) {
    throw new RuntimeException(String.format("Cannot upload configurations to zk. (collection: %s, config set folder: %s)",
      solrPropsConfig.getCollection(), solrPropsConfig.getConfigSetFolder()), e);
  }
  return reloadCollectionNeeded;
}
 

public CloudSolrClient createCloudSolrClient(String zookeeperConnectionString) {
  NoOpResponseParser responseParser = new NoOpResponseParser();
  responseParser.setWriterType("json");

  ConnectStringParser parser = new ConnectStringParser(zookeeperConnectionString);

  CloudSolrClient.Builder cloudBuilder = new CloudSolrClient.Builder(
      parser.getServerAddresses().stream()
          .map(address -> String.format(Locale.ROOT, "%s:%s", address.getHostString(), address.getPort()))
          .collect(Collectors.toList()),
      Optional.ofNullable(parser.getChrootPath()));

  cloudBuilder.withConnectionTimeout(settings.getHttpConnectionTimeout())
      .withSocketTimeout(settings.getHttpReadTimeout());

  CloudSolrClient client = cloudBuilder.build();
  client.setParser(responseParser);

  client.connect();

  return client;
}
 

/**
 * Asserts that the collection has the correct number of shards and replicas
 */
protected void assertCollectionExpectations(String collectionName) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ClusterState clusterState = client.getZkStateReader().getClusterState();

    assertTrue("Could not find new collection " + collectionName, clusterState.hasCollection(collectionName));
    Map<String, Slice> shards = clusterState.getCollection(collectionName).getSlicesMap();
    // did we find expectedSlices shards/shards?
    assertEquals("Found new collection " + collectionName + ", but mismatch on number of shards.", shardCount, shards.size());
    int totalShards = 0;
    for (String shardName : shards.keySet()) {
      totalShards += shards.get(shardName).getReplicas().size();
    }
    int expectedTotalShards = shardCount * replicationFactor;
    assertEquals("Found new collection " + collectionName + " with correct number of shards, but mismatch on number " +
        "of shards.", expectedTotalShards, totalShards);
  } finally {
    client.close();
  }
}
 

@BeforeClass
public static void startCluster() throws Exception {
  File solrXml = new File("src/test/resources/solr.xml");

  tempDir = Files.createTempDirectory("MiniSolrCloudCluster");

  try {
    cluster = new MiniSolrCloudCluster(1, null, tempDir, MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML, null, null);
  } catch (Exception exc) {
    log.error("Failed to initialize a MiniSolrCloudCluster due to: " + exc, exc);
    throw exc;
  }

  cloudSolrClient = new CloudSolrClient(cluster.getZkServer().getZkAddress(), true);
  cloudSolrClient.connect();

  assertTrue(!cloudSolrClient.getZkStateReader().getClusterState().getLiveNodes().isEmpty());
}
 

private SolrClient createSolrClient() {
    if(logger.isDebugEnabled()) {
        logger.debug("HttpClientBuilder = {}", HttpClientUtil.getHttpClientBuilder(), new Exception());
    }
    final ModifiableSolrParams clientParams = new ModifiableSolrParams();
    SolrClient solrClient = null;

    Mode mode = Mode.parse(configuration.get(SOLR_MODE));
    switch (mode) {
        case CLOUD:
            final CloudSolrClient cloudServer = new CloudSolrClient.Builder()
                    .withLBHttpSolrClientBuilder(
                            new LBHttpSolrClient.Builder()
                                    .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams))
                                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    )
                    .withZkHost(getZookeeperURLs(configuration))
                    .sendUpdatesOnlyToShardLeaders()
                    .build();
            cloudServer.connect();
            solrClient = cloudServer;
            logger.info("Created solr client using Cloud based configuration.");
            break;
        case HTTP:
            clientParams.add(HttpClientUtil.PROP_ALLOW_COMPRESSION, configuration.get(HTTP_ALLOW_COMPRESSION).toString());
            clientParams.add(HttpClientUtil.PROP_CONNECTION_TIMEOUT, configuration.get(HTTP_CONNECTION_TIMEOUT).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS_PER_HOST, configuration.get(HTTP_MAX_CONNECTIONS_PER_HOST).toString());
            clientParams.add(HttpClientUtil.PROP_MAX_CONNECTIONS, configuration.get(HTTP_GLOBAL_MAX_CONNECTIONS).toString());
            final HttpClient client = HttpClientUtil.createClient(clientParams);
            solrClient = new LBHttpSolrClient.Builder()
                    .withHttpClient(client)
                    .withBaseSolrUrls(configuration.get(HTTP_URLS))
                    .build();
            logger.info("Created solr client using HTTP based configuration.");
            break;
        default:
            throw new IllegalArgumentException("Unsupported Solr operation mode: " + mode);
    }
    return solrClient;
}
 

protected void waitToSeeLiveNodes(int maxWaitSecs, String zkHost, int numNodes) {
  CloudSolrClient cloudClient = null;
  try {
    cloudClient = new CloudSolrClient.Builder(Collections.singletonList(zkHost), Optional.empty())
        .build();
    cloudClient.connect();
    Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
    int numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0;
    long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(maxWaitSecs, TimeUnit.SECONDS);
    while (System.nanoTime() < timeout && numLiveNodes < numNodes) {
      echo("\nWaiting up to "+maxWaitSecs+" seconds to see "+
          (numNodes-numLiveNodes)+" more nodes join the SolrCloud cluster ...");
      try {
        Thread.sleep(2000);
      } catch (InterruptedException ie) {
        Thread.interrupted();
      }
      liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
      numLiveNodes = (liveNodes != null) ? liveNodes.size() : 0;
    }
    if (numLiveNodes < numNodes) {
      echo("\nWARNING: Only "+numLiveNodes+" of "+numNodes+
          " are active in the cluster after "+maxWaitSecs+
          " seconds! Please check the solr.log for each node to look for errors.\n");
    }
  } catch (Exception exc) {
    CLIO.err("Failed to see if "+numNodes+" joined the SolrCloud cluster due to: "+exc);
  } finally {
    if (cloudClient != null) {
      try {
        cloudClient.close();
      } catch (Exception ignore) {}
    }
  }
}
 

private void waitForCollectionToDisappear(String collection) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    AbstractDistribZkTestBase.waitForCollectionToDisappear(collection, zkStateReader, true, 15);
  } finally {
    client.close();
  }
}
 

private void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception {
  CloudSolrClient client = this.createCloudClient(null);
  try {
    client.connect();
    ZkStateReader zkStateReader = client.getZkStateReader();
    super.waitForRecoveriesToFinish(collection, zkStateReader, verbose);
  } finally {
    client.close();
  }
}
 

private SolrCloudScraper createSolrCloudScraper() {
  CloudSolrClient solrClient = new CloudSolrClient.Builder(
      Collections.singletonList(cluster.getZkServer().getZkAddress()), Optional.empty())
      .build();

  NoOpResponseParser responseParser = new NoOpResponseParser();
  responseParser.setWriterType("json");

  solrClient.setParser(responseParser);

  solrClient.connect();

  SolrClientFactory factory = new SolrClientFactory(PrometheusExporterSettings.builder().build());

  return new SolrCloudScraper(solrClient, executor, factory);
}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

/**
 * Creates the cloud.
 */
private static void createCloud() {
  Path dataPath = Paths.get("src" + File.separator + "test" + File.separator
      + "resources" + File.separator + "data");
  String solrxml = MiniSolrCloudCluster.DEFAULT_CLOUD_SOLR_XML;
  JettyConfig jettyConfig = JettyConfig.builder().setContext("/solr").build();
  File cloudBase = Files.createTempDir();
  cloudBaseDir = cloudBase.toPath();
  // create subdirectories
  Path clusterDir = cloudBaseDir.resolve("cluster");
  Path logDir = cloudBaseDir.resolve("log");
  if (clusterDir.toFile().mkdir() && logDir.toFile().mkdir()) {
    // set log directory
    System.setProperty("solr.log.dir", logDir.toAbsolutePath().toString());
    try {
      cloudCluster = new MiniSolrCloudCluster(1, clusterDir, solrxml,
          jettyConfig);
      CloudSolrClient client = cloudCluster.getSolrClient();
      client.connect();
      createCloudCollection(COLLECTION_ALL_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_ALL_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART1_OPTIMIZED, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_PART2_MULTIPLE_SEGMENTS, 1, 1,
          dataPath.resolve("conf"));
      createCloudCollection(COLLECTION_DISTRIBUTED, 1, 1,
          dataPath.resolve("conf"));

      // collection1
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(1));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(2));
      client.add(COLLECTION_ALL_OPTIMIZED, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_OPTIMIZED);
      client.optimize(COLLECTION_ALL_OPTIMIZED);
      // collection2
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(1));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_ALL_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_ALL_MULTIPLE_SEGMENTS);
      // collection3
      client.add(COLLECTION_PART1_OPTIMIZED, solrDocuments.get(1));
      client.commit(COLLECTION_PART1_OPTIMIZED);
      // collection4
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(2));
      client.add(COLLECTION_PART2_MULTIPLE_SEGMENTS, solrDocuments.get(3));
      client.commit(COLLECTION_PART2_MULTIPLE_SEGMENTS);
    } catch (Exception e) {
      e.printStackTrace();
      log.error(e);
    }
  } else {
    log.error("couldn't create directories");
  }
}
 

/**
 * Returns the list of shards of the default collection.
 *
 * @param zkHost            ZooKeeper URL
 * @param chronixCollection Solr collection name for chronix time series data
 * @return the list of shards of the default collection
 */
public List<String> getShardList(String zkHost, String chronixCollection) throws IOException {

    CloudSolrClient cloudSolrClient = new CloudSolrClient(zkHost);
    List<String> shards = new ArrayList<>();

    try {
        cloudSolrClient.connect();

        ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

        ClusterState clusterState = zkStateReader.getClusterState();

        String[] collections;
        if (clusterState.hasCollection(chronixCollection)) {
            collections = new String[]{chronixCollection};
        } else {
            // might be a collection alias?
            Aliases aliases = zkStateReader.getAliases();
            String aliasedCollections = aliases.getCollectionAlias(chronixCollection);
            if (aliasedCollections == null)
                throw new IllegalArgumentException("Collection " + chronixCollection + " not found!");
            collections = aliasedCollections.split(",");
        }

        Set<String> liveNodes = clusterState.getLiveNodes();
        Random random = new Random(5150);

        for (String coll : collections) {
            for (Slice slice : clusterState.getSlices(coll)) {
                List<String> replicas = new ArrayList<>();
                for (Replica r : slice.getReplicas()) {
                    if (r.getState().equals(Replica.State.ACTIVE)) {
                        ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(r);
                        if (liveNodes.contains(replicaCoreProps.getNodeName()))
                            replicas.add(replicaCoreProps.getCoreUrl());
                    }
                }
                int numReplicas = replicas.size();
                if (numReplicas == 0)
                    throw new IllegalStateException("Shard " + slice.getName() + " in collection " +
                            coll + " does not have any active replicas!");

                String replicaUrl = (numReplicas == 1) ? replicas.get(0) : replicas.get(random.nextInt(replicas.size()));
                shards.add(replicaUrl);
            }
        }
    } finally {
        cloudSolrClient.close();
    }

    return shards;
}