Java Code Examples for org.apache.solr.common.cloud.ZkStateReader#NODE_NAME_PROP

void deleteCoreNode(String collectionName, String replicaName, Replica replica, String core) throws Exception {
  ZkNodeProps m = new ZkNodeProps(
      Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(),
      ZkStateReader.CORE_NAME_PROP, core,
      ZkStateReader.NODE_NAME_PROP, replica.getStr(ZkStateReader.NODE_NAME_PROP),
      ZkStateReader.COLLECTION_PROP, collectionName,
      ZkStateReader.CORE_NODE_NAME_PROP, replicaName,
      ZkStateReader.BASE_URL_PROP, replica.getStr(ZkStateReader.BASE_URL_PROP));
  overseer.offerStateUpdate(Utils.toJSON(m));
}
 

public void createCollection(String collection) throws Exception {
  ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
      CollectionParams.CollectionAction.CREATE.toLower(), ZkStateReader.NODE_NAME_PROP, getNodeName(),
      ZkStateReader.COLLECTION_PROP, collection);
  overseerJobQueue.offer(Utils.toJSON(m));
}
 

@Test
public void downNodeReportsAllImpactedCollectionsAndNothingElse() throws IOException {
  NodeMutator nm = new NodeMutator();

  //We use 2 nodes with maxShardsPerNode as 1
  //Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2
  //Collection2: 1 shard X 1 replica = replica1 on node2
  ZkStateReader reader = ClusterStateMockUtil.buildClusterState("csrr2rDcsr2", 1, 1, NODE1, NODE2);
  ClusterState clusterState = reader.getClusterState();
  assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL);
  assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL);
  assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL);

  ZkNodeProps props = new ZkNodeProps(ZkStateReader.NODE_NAME_PROP, NODE1);
  List<ZkWriteCommand> writes = nm.downNode(clusterState, props);
  assertEquals(writes.size(), 1);
  assertEquals(writes.get(0).name, "collection1");
  assertEquals(writes.get(0).collection.getReplica("replica1").getState(), Replica.State.DOWN);
  assertEquals(writes.get(0).collection.getReplica("replica2").getState(), Replica.State.ACTIVE);
  reader.close();

  //We use 3 nodes with maxShardsPerNode as 1
  //Collection1: 2 shards X 1 replica = replica1 on node1 and replica2 on node2
  //Collection2: 1 shard X 1 replica = replica1 on node2
  //Collection3: 1 shard X 3 replica = replica1 on node1 , replica2 on node2, replica3 on node3
  reader = ClusterStateMockUtil.buildClusterState("csrr2rDcsr2csr1r2r3", 1, 1, NODE1, NODE2, NODE3);
  clusterState = reader.getClusterState();
  assertEquals(clusterState.getCollection("collection1").getReplica("replica1").getBaseUrl(), NODE1_URL);
  assertEquals(clusterState.getCollection("collection1").getReplica("replica2").getBaseUrl(), NODE2_URL);

  assertEquals(clusterState.getCollection("collection2").getReplica("replica4").getBaseUrl(), NODE2_URL);

  assertEquals(clusterState.getCollection("collection3").getReplica("replica5").getBaseUrl(), NODE1_URL);
  assertEquals(clusterState.getCollection("collection3").getReplica("replica6").getBaseUrl(), NODE2_URL);
  assertEquals(clusterState.getCollection("collection3").getReplica("replica7").getBaseUrl(), NODE3_URL);

  writes = nm.downNode(clusterState, props);
  assertEquals(writes.size(), 2);
  for (ZkWriteCommand write : writes) {
    if (write.name.equals("collection1")) {
      assertEquals(write.collection.getReplica("replica1").getState(), Replica.State.DOWN);
      assertEquals(write.collection.getReplica("replica2").getState(), Replica.State.ACTIVE);
    } else if (write.name.equals("collection3")) {
      assertEquals(write.collection.getReplica("replica5").getState(), Replica.State.DOWN);
      assertEquals(write.collection.getReplica("replica6").getState(), Replica.State.ACTIVE);
      assertEquals(write.collection.getReplica("replica7").getState(), Replica.State.ACTIVE);
    } else {
      fail("No other collection needs to be changed");
    }
  }
  reader.close();
}
 

@Slow
@LogLevel(value = "org.apache.solr.cloud=DEBUG;org.apache.solr.cloud.overseer=DEBUG")
public void testPublishAndWaitForDownStates() throws Exception  {

  /*
  This test asserts that if zkController.publishAndWaitForDownStates uses only core name to check if all local
  cores are down then the method will return immediately but if it uses coreNodeName (as it does after SOLR-6665 then
  the method will timeout).
  We setup the cluster state in such a way that two replicas with same core name exist on non-existent nodes
  and core container also has a dummy core that has the same core name. The publishAndWaitForDownStates before SOLR-6665
  would only check the core names and therefore return immediately but after SOLR-6665 it should time out.
   */

  assumeWorkingMockito();
  final String collectionName = "testPublishAndWaitForDownStates";
  Path zkDir = createTempDir(collectionName);
  CoreContainer cc = null;

  String nodeName = "127.0.0.1:8983_solr";

  ZkTestServer server = new ZkTestServer(zkDir);
  try {
    server.run();

    AtomicReference<ZkController> zkControllerRef = new AtomicReference<>();
    cc = new MockCoreContainer()  {
      @Override
      public List<CoreDescriptor> getCoreDescriptors() {
        CoreDescriptor descriptor = new CoreDescriptor(collectionName, TEST_PATH(), Collections.emptyMap(), new Properties(), zkControllerRef.get());
        // non-existent coreNodeName, this will cause zkController.publishAndWaitForDownStates to wait indefinitely
        // when using coreNodeName but usage of core name alone will return immediately
        descriptor.getCloudDescriptor().setCoreNodeName("core_node0");
        return Collections.singletonList(descriptor);
      }
    };
    ZkController zkController = null;

    try {
      CloudConfig cloudConfig = new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "solr").build();
      zkController = new ZkController(cc, server.getZkAddress(), TIMEOUT, cloudConfig, () -> null);
      zkControllerRef.set(zkController);

      zkController.getZkClient().makePath(ZkStateReader.getCollectionPathRoot(collectionName), new byte[0], CreateMode.PERSISTENT, true);

      ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION,
          CollectionParams.CollectionAction.CREATE.toLower(), ZkStateReader.NODE_NAME_PROP, nodeName, ZkStateReader.NUM_SHARDS_PROP, "1",
          "name", collectionName);
      zkController.getOverseerJobQueue().offer(Utils.toJSON(m));

      HashMap<String, Object> propMap = new HashMap<>();
      propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower());
      propMap.put(COLLECTION_PROP, collectionName);
      propMap.put(SHARD_ID_PROP, "shard1");
      propMap.put(ZkStateReader.NODE_NAME_PROP, "non_existent_host1");
      propMap.put(ZkStateReader.CORE_NAME_PROP, collectionName);
      propMap.put(ZkStateReader.STATE_PROP, "active");
      zkController.getOverseerJobQueue().offer(Utils.toJSON(propMap));

      propMap = new HashMap<>();
      propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower());
      propMap.put(COLLECTION_PROP, collectionName);
      propMap.put(SHARD_ID_PROP, "shard1");
      propMap.put(ZkStateReader.NODE_NAME_PROP, "non_existent_host2");
      propMap.put(ZkStateReader.CORE_NAME_PROP, collectionName);
      propMap.put(ZkStateReader.STATE_PROP, "down");
      zkController.getOverseerJobQueue().offer(Utils.toJSON(propMap));

      zkController.getZkStateReader().forciblyRefreshAllClusterStateSlow();

      long now = System.nanoTime();
      long timeout = now + TimeUnit.NANOSECONDS.convert(5, TimeUnit.SECONDS);
      zkController.publishAndWaitForDownStates(5);
      assertTrue("The ZkController.publishAndWaitForDownStates should have timed out but it didn't", System.nanoTime() >= timeout);
    } finally {
      if (zkController != null)
        zkController.close();
    }
  } finally {
    if (cc != null) {
      cc.shutdown();
    }
    server.shutdown();
  }
}
 

@Test
public void deleteReplicaFromClusterState() throws Exception {
  final String collectionName = "deleteFromClusterStateCollection";
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 3)
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 3);
  
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "1"));
  cluster.getSolrClient().add(collectionName, new SolrInputDocument("id", "2"));
  cluster.getSolrClient().commit(collectionName);

  cluster.waitForActiveCollection(collectionName, 1, 3);

  Slice shard = getCollectionState(collectionName).getSlice("shard1");

  // don't choose the leader to shutdown, it just complicates things unnecessarily
  Replica replica = getRandomReplica(shard, (r) ->
                                     ( r.getState() == Replica.State.ACTIVE &&
                                       ! r.equals(shard.getLeader())));
  
  JettySolrRunner replicaJetty = cluster.getReplicaJetty(replica);
  ZkStateReaderAccessor accessor = new ZkStateReaderAccessor(replicaJetty.getCoreContainer().getZkController().getZkStateReader());

  final long preDeleteWatcherCount = countUnloadCoreOnDeletedWatchers
    (accessor.getStateWatchers(collectionName));

  ZkNodeProps m = new ZkNodeProps(
      Overseer.QUEUE_OPERATION, OverseerAction.DELETECORE.toLower(),
      ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
      ZkStateReader.NODE_NAME_PROP, replica.getNodeName(),
      ZkStateReader.COLLECTION_PROP, collectionName,
      ZkStateReader.CORE_NODE_NAME_PROP, replica.getName(),
      ZkStateReader.BASE_URL_PROP, replica.getBaseUrl());

  cluster.getOpenOverseer().getStateUpdateQueue().offer(Utils.toJSON(m));

  waitForState("Timeout waiting for replica get deleted", collectionName,
      (liveNodes, collectionState) -> collectionState.getSlice("shard1").getReplicas().size() == 2);

  TimeOut timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  timeOut.waitFor("Waiting for replica get unloaded", () ->
      replicaJetty.getCoreContainer().getCoreDescriptor(replica.getCoreName()) == null
  );
  
  // the core should no longer have a watch collection state since it was removed
  timeOut = new TimeOut(60, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  timeOut.waitFor("Waiting for core's watcher to be removed", () -> {
      final long postDeleteWatcherCount = countUnloadCoreOnDeletedWatchers
        (accessor.getStateWatchers(collectionName));
      log.info("preDeleteWatcherCount={} vs postDeleteWatcherCount={}",
               preDeleteWatcherCount, postDeleteWatcherCount);
      return (preDeleteWatcherCount - 1L == postDeleteWatcherCount);
    });
  
  CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient());
}
 

@Test
@SuppressWarnings({"try"})
public void testDownNodeFailover() throws Exception {
  MockZKController mockController = null;
  SolrZkClient overseerClient = null;

  try {

    ZkController.createClusterZkNodes(zkClient);

    overseerClient = electNewOverseer(server.getZkAddress());

    try (ZkStateReader reader = new ZkStateReader(zkClient)) {
      reader.createClusterStateWatchersAndUpdate();

      mockController = new MockZKController(server.getZkAddress(), "127.0.0.1", overseers);

      try (ZkController zkController = createMockZkController(server.getZkAddress(), zkClient, reader)) {

        for (int i = 0; i < 5; i++) {
          mockController.createCollection("collection" + i, 1);
          assertNotNull("shard got no id?", mockController.publishState("collection" + i, "core1",
              "core_node1", "shard1", Replica.State.ACTIVE, 1, true, overseers.get(0)));
        }
      }
      ZkNodeProps m = new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.DOWNNODE.toLower(),
          ZkStateReader.NODE_NAME_PROP, "127.0.0.1");
      List<ZkWriteCommand> commands = new NodeMutator().downNode(reader.getClusterState(), m);

      ZkDistributedQueue q = overseers.get(0).getStateUpdateQueue();

      q.offer(Utils.toJSON(m));

      verifyReplicaStatus(reader, commands.get(0).name, "shard1", "core_node1", Replica.State.DOWN);
      overseerClient.close();

      overseerClient = electNewOverseer(server.getZkAddress());
      for (int i = 0; i < 5; i++) {
        verifyReplicaStatus(reader, "collection" + i, "shard1", "core_node1", Replica.State.DOWN);
      }
    }
  } finally {
    if (mockController != null) {
      mockController.close();
    }
    close(overseerClient);
  }
}