Java Code Examples for org.apache.solr.client.solrj.embedded.JettySolrRunner#getNodeName()

@SuppressWarnings("WeakerAccess")
Set<String> getLeaderCoreNames(ClusterState clusterState) {
  Set<String> leaders = new TreeSet<>(); // sorted just to make it easier to read when debugging...
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (JettySolrRunner jettySolrRunner : jettySolrRunners) {
    List<CoreDescriptor> coreDescriptors = jettySolrRunner.getCoreContainer().getCoreDescriptors();
    for (CoreDescriptor core : coreDescriptors) {
      String nodeName = jettySolrRunner.getNodeName();
      String collectionName = core.getCollectionName();
      DocCollection collectionOrNull = clusterState.getCollectionOrNull(collectionName);
      List<Replica> leaderReplicas = collectionOrNull.getLeaderReplicas(nodeName);
      if (leaderReplicas != null) {
        for (Replica leaderReplica : leaderReplicas) {
          leaders.add(leaderReplica.getCoreName());
        }
      }
    }
  }
  return leaders;
}
 

private String addNode() throws Exception {
  JettySolrRunner solr = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  String nodeId = solr.getNodeName();
  if (simulated) {
    ((SimCloudManager) cloudManager).getSimClusterStateProvider().simAddNode(nodeId);
  }
  return nodeId;
}
 

public static String getNonLeaderNode(MiniSolrCloudCluster cluster, String collection) throws Exception {
  String leaderNode = getLeaderNode(cluster, collection);
  for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
    if (!jetty.getNodeName().equals(leaderNode)) {
      return jetty.getNodeName();
    }
  }
  return cluster.getJettySolrRunners().get(0).getNodeName();
}
 

public void testCreateCollectionAddReplica() throws Exception  {
  final JettySolrRunner jetty = cluster.getRandomJetty(random());
  final String jettyNodeName = jetty.getNodeName();
  final int port = jetty.getLocalPort();

  final String commands =  "{set-policy :{c1 : [{replica:0 , shard:'#EACH', port: '!" + port + "'}]}}";
  cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));

  final String collectionName = "testCreateCollectionAddReplica";
  log.info("Creating collection {}", collectionName);
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1)
      .setPolicy("c1")
      .process(cluster.getSolrClient());

  waitForState("Should have found exactly one replica, only on expected jetty: " +
               jettyNodeName + "/" + port,
               collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 1),
               120, TimeUnit.SECONDS);

  log.info("Adding replica to {}", collectionName);
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
    .process(cluster.getSolrClient());
  
  waitForState("Should have found exactly two replicas, only on expected jetty: " +
               jettyNodeName + "/" + port,
               collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 2),
               120, TimeUnit.SECONDS);

}
 

@Test
public void testLeaderUnloaded() throws Exception {
  CloudSolrClient solrClient = cluster.getSolrClient();

  String collectionName = "testLeaderUnloaded";
  CollectionAdminRequest.createCollection(collectionName, 1, 2)
      .process(solrClient);

  waitForState("Expected collection: testLeaderUnloaded to be live with 1 shard and 2 replicas",
      collectionName, clusterShape(1, 2));

  JettySolrRunner newNode = cluster.startJettySolrRunner();
  String newNodeName = newNode.getNodeName();

  // add a replica to the new node so that it starts watching the collection
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
      .setNode(newNodeName)
      .process(solrClient);

  // now delete the leader
  Replica leader = solrClient.getZkStateReader().getLeaderRetry(collectionName, "shard1");
  CollectionAdminRequest.deleteReplica(collectionName, "shard1", leader.getName())
      .process(solrClient);

  // add another replica to the new node. When it starts recovering, it will likely have stale state
  // and ask the erstwhile leader to PREPRECOVERY which will hang for about 30 seconds
  CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
      .setNode(newNodeName)
      .process(solrClient);

  // in the absence of the fixes made in SOLR-10914, this statement will timeout after 90s
  waitForState("Expected collection: testLeaderUnloaded to be live with 1 shard and 3 replicas",
      collectionName, clusterShape(1, 3));
}
 

private JettySolrRunner getJettyForReplica(Replica replica) {
  for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
    String nodeName = jetty.getNodeName();
    if (nodeName != null && nodeName.equals(replica.getNodeName())) return jetty;
  }
  throw new IllegalArgumentException("Can not find jetty for replica "+ replica);
}
 

@Test
public void testAddReplicaSimple() throws Exception {
  String prefix = "testAddReplica";
  String xyz = prefix + "_xyz";
  String abc = prefix + "_abc";

  CloudSolrClient solrClient = cluster.getSolrClient();
  String chosenNode = cluster.getRandomJetty(random()).getNodeName();
  log.info("Chosen node {} for collection {}", chosenNode, abc);
  CollectionAdminRequest.createCollection(abc, 1, 1)
      .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
      .process(solrClient);
  CollectionAdminRequest.createCollection(xyz, 1, 1)
      .setWithCollection(abc)
      .process(solrClient);

  String otherNode = null;
  for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
    if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
      otherNode = jettySolrRunner.getNodeName();
    }
  }
  CollectionAdminRequest.addReplicaToShard(xyz, "shard1")
      .setNode(otherNode)
      .process(solrClient);
  DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
  DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);

  assertTrue(collection.getReplicas().stream().noneMatch(replica -> withCollection.getReplicas(replica.getNodeName()).isEmpty()));
}
 

private void testDoRecoveryOnRestart() throws Exception {
  String testCollectionName = "collDoRecoveryOnRestart";
  try {
    // Inject pausing in recovery op, hence the replica won't be able to finish recovery

    TestInjection.prepRecoveryOpPauseForever = "true:100";
    
    createCollection(testCollectionName, "conf1", 1, 2, 1);
    cloudClient.setDefaultCollection(testCollectionName);

    sendDoc(1, 2);

    JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
    List<Replica> notLeaders =
        ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 1);

    SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
    SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));

    proxy0.close();
    leaderProxy.close();

    // indexing during a partition
    int achievedRf = sendDoc(2, 1, leaderJetty);
    assertEquals("Unexpected achieved replication factor", 1, achievedRf);
    try (ZkShardTerms zkShardTerms = new ZkShardTerms(testCollectionName, "shard1", cloudClient.getZkStateReader().getZkClient())) {
      assertFalse(zkShardTerms.canBecomeLeader(notLeaders.get(0).getName()));
    }
    waitForState(testCollectionName, notLeaders.get(0).getName(), DOWN, 10000);

    // heal partition
    proxy0.reopen();
    leaderProxy.reopen();

    waitForState(testCollectionName, notLeaders.get(0).getName(), RECOVERING, 10000);

    System.clearProperty("solrcloud.skip.autorecovery");
    JettySolrRunner notLeaderJetty = getJettyOnPort(getReplicaPort(notLeaders.get(0)));
    String notLeaderNodeName = notLeaderJetty.getNodeName();
    notLeaderJetty.stop();
    
    cloudClient.getZkStateReader().waitForLiveNodes(15, TimeUnit.SECONDS, SolrCloudTestCase.missingLiveNode(notLeaderNodeName));

    notLeaderJetty.start();
    ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, 130);
    assertDocsExistInAllReplicas(notLeaders, testCollectionName, 1, 2);
  } finally {
    TestInjection.prepRecoveryOpPauseForever = null;
    TestInjection.notifyPauseForeverDone();
  }

  // try to clean up
  attemptCollectionDelete(cloudClient, testCollectionName);
}
 

@Test
@SuppressWarnings({"unchecked"})
public void testNodeAddedTriggerRestoreState() throws Exception {
  
  final String triggerName = "node_added_restore_trigger";

  // should be enough to ensure trigger doesn't fire any actions until we replace the trigger
  waitForSeconds = 500000;
  CloudTestUtils.assertAutoScalingRequest
    (cloudManager,
     "{" +
     "'set-trigger' : {" +
     "'name' : '"+triggerName+"'," +
     "'event' : 'nodeAdded'," +
     "'waitFor' : '"+waitForSeconds+"s'," + 
     "'enabled' : true," +
     "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
     "}}");
  
  assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
             actionInitCalled.await(60, TimeUnit.SECONDS));

  // start a new node
  final JettySolrRunner newNode = cluster.startJettySolrRunner();
  final String nodeName = newNode.getNodeName();

  // poll the internal state of the trigger until it run()s at least once and updates
  // it's internal state to know the node we added is live
  //
  // (this should run roughly once a second)
  (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource()))
  .waitFor("initial trigger never ran to detect new live node", () ->
           (((Collection<String>) getTriggerState(triggerName).get("lastLiveNodes"))
            .contains(nodeName)));
  
  // since we know the nodeAdded event has been detected, we can recored the current timestamp
  // (relative to the cluster's time source) and later assert that (restored state) correctly
  // tracked that the event happened prior to "now"
  final long maxEventTimeNs = cloudManager.getTimeSource().getTimeNs();
  
  //
  // now replace the trigger with a new instance to test that the state gets copied over correctly
  //
  
  // reset the actionInitCalled counter so we can confirm the second instances is inited
  actionInitCalled = new CountDownLatch(1);
  // use a low waitTime to ensure it processes the event quickly.
  // (this updated property also ensures the set-trigger won't be treated as a No-Op)
  waitForSeconds = 0 + random().nextInt(3);
  CloudTestUtils.assertAutoScalingRequest
    (cloudManager,
     "{" +
     "'set-trigger' : {" +
     "'name' : '"+triggerName+"'," +
     "'event' : 'nodeAdded'," +
     "'waitFor' : '"+waitForSeconds+"s'," + 
     "'enabled' : true," +
     "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
     "}}");
  
  assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
             actionInitCalled.await(60, TimeUnit.SECONDS));

  // the trigger actions should now (eventually) record that the node was added
  assertTrue("Second instance of our trigger never fired the action to process the event",
             triggerFiredLatch.await(30, TimeUnit.SECONDS));
  
  assertEquals("Wrong number of events recorded: " + events.toString(),
               1, events.size());
  
  final TriggerEvent event = events.iterator().next();
  assertNotNull("null event???", event);
  assertTrue("Event should have been a nodeAdded event: " + event.getClass(),
             event instanceof NodeAddedTrigger.NodeAddedEvent);

  assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES));
  assertEquals("event has incorrect NODE_NAMES: " + event,
               Collections.singletonList(nodeName),
               event.getProperty(TriggerEvent.NODE_NAMES));
  
  assertTrue("event TS is too late, should be before (max) expected TS @ "
             + maxEventTimeNs + ": " + event,
             event.getEventTime() < maxEventTimeNs);
  
  assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES));
  assertEquals("event has unexpeted number of EVENT_TIMES: " + event,
               1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size());
  assertEquals("event's TS doesn't match EVENT_TIMES: " + event,
               event.getEventTime(),
               ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next());
}
 

@Test
  //Commented 14-Oct-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 23-Aug-2018
  public void testMoveReplicaWithCollection() throws Exception {
    String prefix = "testMoveReplicaWithCollection";
    String xyz = prefix + "_xyz";
    String abc = prefix + "_abc";

    CloudSolrClient solrClient = cluster.getSolrClient();

    String setClusterPolicyCommand = "{" +
        " 'set-cluster-policy': [" +
        "      {'cores':'<10', 'node':'#ANY'}," +
        "      {'replica':'<2', 'node':'#ANY'}," +
        "    ]" +
        "}";
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);

    String chosenNode = cluster.getRandomJetty(random()).getNodeName();
    log.info("Chosen node {} for collection {}", chosenNode, abc);
    CollectionAdminRequest.createCollection(abc, 1, 1)
        .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
        .process(solrClient);
    CollectionAdminRequest.createCollection(xyz, 1, 1)
        .setWithCollection(abc)
        .process(solrClient);

    DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    assertEquals(chosenNode, collection.getReplicas().iterator().next().getNodeName());

    String otherNode = null;
    for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
      if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
        otherNode = jettySolrRunner.getNodeName();
      }
    }

    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);
    assertNull(collection.getReplicas(otherNode)); // sanity check
    assertNull(withCollection.getReplicas(otherNode)); // sanity check

    try {
      new CollectionAdminRequest.MoveReplica(abc, collection.getReplicas().iterator().next().getName(), otherNode)
          .process(solrClient);
      fail("Expected moving a replica of 'withCollection': " + abc + " to fail");
    } catch (BaseHttpSolrClient.RemoteSolrException e) {
      assertTrue(e.getMessage().contains("Collection: testMoveReplicaWithCollection_abc is co-located with collection: testMoveReplicaWithCollection_xyz"));
    }
//    zkClient().printLayoutToStdOut();
    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh
    DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh

    // sanity check that the failed move operation didn't actually change our co-location guarantees
    assertTrue(collection.getReplicas().stream().noneMatch(
        replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null
            || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty()));
  }
 

/**
 * Test that basic autoAddReplicaLogic kicks in when a node is lost 
 */
@Test
public void testSimple() throws Exception {
  final String COLLECTION = "test_simple";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(true)
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);
  
  // start the tests
  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  waitForNodeLeave(lostNodeName);
  
  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
               
  checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));

}
 

/**
 * Test that basic autoAddReplicaLogic logic is <b>not</b> used if the cluster prop for it is disabled 
 * (even if sys prop is set after collection is created)
 */
@Test
public void testClusterPropOverridesCollecitonProp() throws Exception {
  final String COLLECTION = "test_clusterprop";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);

  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(true)
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);

  // check cluster property is considered
  disableAutoAddReplicasInCluster();

  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);

  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  
  waitForNodeLeave(lostNodeName);
  
  waitForState(COLLECTION + "=(2,2)", COLLECTION,
               clusterShape(2, 2), 90, TimeUnit.SECONDS);
               

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
  
  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);

}
 

/**
 * Test that we can modify a collection after creation to add autoAddReplicas.
 */
@Test
public void testAddCollectionPropAfterCreation() throws Exception {
  final String COLLECTION = "test_addprop";
  final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
  final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
  final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);

  if (log.isInfoEnabled()) {
    log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
        jetty1.getNodeName(), jetty1.getLocalPort(),
        jetty2.getNodeName(), jetty2.getLocalPort());
  }
           
  CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
    .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
    .setAutoAddReplicas(false) // NOTE: false
    .setMaxShardsPerNode(2)
    .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(COLLECTION, 2, 4);
  
  log.info("Modifying {} to use autoAddReplicas", COLLECTION);
  new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
    @Override
    public SolrParams getParams() {
      ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
      params.set("collection", COLLECTION);
      params.set("autoAddReplicas", true);
      return params;
    }
  }.process(cluster.getSolrClient());

  JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
  String lostNodeName = lostJetty.getNodeName();
  List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);

  if (log.isInfoEnabled()) {
    log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.stop();
  
  cluster.waitForJettyToStop(lostJetty);
  
  waitForNodeLeave(lostNodeName);

  waitForState(COLLECTION + "=(2,4) w/o down replicas",
               COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
  checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);

  if (log.isInfoEnabled()) {
    log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
  }
  lostJetty.start();
  
  waitForNodeLive(lostJetty);
  
  assertTrue("Timeout waiting for all live and active",
             ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
}
 

@Test
public void testRestoreState() throws Exception {
  CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
  long waitForSeconds = 1 + random().nextInt(5);
  Map<String, Object> props = createTriggerProps(waitForSeconds);

  JettySolrRunner newNode = cluster.startJettySolrRunner();
  cluster.waitForAllNodes(30);
  String lostNodeName = newNode.getNodeName();

  // remove a node but update the trigger before the waitFor period expires
  // and assert that the new trigger still fires

  NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger");
  trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
  trigger.init();
  trigger.setProcessor(noFirstRunProcessor);
  trigger.run();

  // stop the newly created node
  List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
  for (int i = 0; i < jettySolrRunners.size(); i++) {
    JettySolrRunner jettySolrRunner = jettySolrRunners.get(i);
    if (newNode == jettySolrRunner) {
      JettySolrRunner j = cluster.stopJettySolrRunner(i);
      cluster.waitForJettyToStop(j);
      break;
    }
  }

  trigger.run(); // this run should detect the lost node
  trigger.close(); // close the old trigger

  try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name"))  {
    newTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
    newTrigger.init();
    try {
      newTrigger.restoreState(trigger);
      fail("Trigger should only be able to restore state from an old trigger of the same name");
    } catch (AssertionError e) {
      // expected
    }
  }

  try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger")) {
    final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
    newTrigger.configure(container.getResourceLoader(), cloudManager, props);
    newTrigger.init();
    AtomicBoolean fired = new AtomicBoolean(false);
    AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
    newTrigger.setProcessor(event -> {
      if (fired.compareAndSet(false, true)) {
        eventRef.set(event);
        long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
        long eventTimeNanos = event.getEventTime();
        long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
        if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
          fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
        }
      } else {
        fail("NodeLostListener was fired more than once!");
      }
      return true;
    });
    newTrigger.restoreState(trigger); // restore state from the old trigger
    int counter = 0;
    do {
      newTrigger.run();
      Thread.sleep(1000);
      if (counter++ > 10) {
        fail("Lost node was not discovered by trigger even after 10 seconds");
      }
    } while (!fired.get());

    TriggerEvent nodeLostEvent = eventRef.get();
    assertNotNull(nodeLostEvent);
    @SuppressWarnings({"unchecked"})
    List<String> nodeNames = (List<String>)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
    assertTrue(nodeNames.contains(lostNodeName));
  }
}
 

@Test
  public void testMoveReplicaMainCollection() throws Exception {
    String prefix = "testMoveReplicaMainCollection";
    String xyz = prefix + "_xyz";
    String abc = prefix + "_abc";

    CloudSolrClient solrClient = cluster.getSolrClient();

    String setClusterPolicyCommand = "{" +
        " 'set-cluster-policy': [" +
        "      {'cores':'<10', 'node':'#ANY'}," +
        "      {'replica':'<2', 'node':'#ANY'}," +
        "    ]" +
        "}";
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);

    String chosenNode = cluster.getRandomJetty(random()).getNodeName();
    log.info("Chosen node {} for collection {}", chosenNode, abc);
    CollectionAdminRequest.createCollection(abc, 1, 1)
        .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
        .process(solrClient);
    CollectionAdminRequest.createCollection(xyz, 1, 1)
        .setWithCollection(abc)
        .process(solrClient);

    String otherNode = null;
    for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
      if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
        otherNode = jettySolrRunner.getNodeName();
      }
    }

    DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);
    assertNull(collection.getReplicas(otherNode)); // sanity check
    assertNull(withCollection.getReplicas(otherNode)); // sanity check

    CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(xyz, collection.getReplicas().iterator().next().getName(), otherNode);
    moveReplica.setWaitForFinalState(true);
    moveReplica.process(solrClient);
//    zkClient().printLayoutToStdOut();
    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh
    DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh
    assertTrue(collection.getReplicas().stream().noneMatch(
        replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null
            || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty()));
  }
 

public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception {
  JettySolrRunner jetty = cluster.getJettySolrRunners().get(0);
  String nrtNodeName = jetty.getNodeName();
  int nrtPort = jetty.getLocalPort();

  jetty = cluster.getJettySolrRunners().get(1);
  String pullNodeName = jetty.getNodeName();
  int pullPort = jetty.getLocalPort();

  jetty = cluster.getJettySolrRunners().get(2);
  String tlogNodeName = jetty.getNodeName();
  int tlogPort = jetty.getLocalPort();
  log.info("NRT {} PULL {} , TLOG {} ", nrtNodeName, pullNodeName, tlogNodeName);

  String commands = "{set-cluster-policy :[" +
      "{replica:0 , shard:'#EACH', type: NRT, port: '!" + nrtPort + "'}" +
      "{replica:0 , shard:'#EACH', type: PULL, port: '!" + pullPort + "'}" +
      "{replica:0 , shard:'#EACH', type: TLOG, port: '!" + tlogPort + "'}" +
      "]}";


  cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
  Map<String, Object> json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, true);
  assertEquals("full json:" + Utils.toJSONString(json), "!" + nrtPort,
      Utils.getObjectByPath(json, true, "cluster-policy[0]/port"));
  assertEquals("full json:" + Utils.toJSONString(json), "!" + pullPort,
      Utils.getObjectByPath(json, true, "cluster-policy[1]/port"));
  assertEquals("full json:" + Utils.toJSONString(json), "!" + tlogPort,
      Utils.getObjectByPath(json, true, "cluster-policy[2]/port"));

  final String collectionName = "addshard_with_reptype_using_policy";
  CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1", 1, 1, 1)
      .setMaxShardsPerNode(-1)
      .process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 1, 3);

  DocCollection coll = getCollectionState(collectionName);


  BiConsumer<String, Replica> verifyReplicas = (s, replica) -> {
    switch (replica.getType()) {
      case NRT: {
        assertTrue("NRT replica should be in " + nrtNodeName, replica.getNodeName().equals(nrtNodeName));
        break;
      }
      case TLOG: {
        assertTrue("TLOG replica should be in " + tlogNodeName, replica.getNodeName().equals(tlogNodeName));
        break;
      }
      case PULL: {
        assertTrue("PULL replica should be in " + pullNodeName, replica.getNodeName().equals(pullNodeName));
        break;
      }
    }

  };
  coll.forEachReplica(verifyReplicas);

  CollectionAdminRequest.createShard(collectionName, "s3").
      process(cluster.getSolrClient());
  
  cluster.waitForActiveCollection(collectionName, 2, 6);
  
  coll = getCollectionState(collectionName);
  assertEquals(3, coll.getSlice("s3").getReplicas().size());
  coll.forEachReplica(verifyReplicas);
}
 

@Test
  public void testExecute() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);
    
    cluster.waitForActiveCollection(collectionName, 1, 2);

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));

    JettySolrRunner sourceNode = cluster.getRandomJetty(random());
    String sourceNodeName = sourceNode.getNodeName();
    ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
        .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
    assertFalse(otherJetties.isEmpty());
    JettySolrRunner survivor = otherJetties.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (zkClient().exists(parentPath, true)) {
              java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
              if (!children.isEmpty()) {
                String child = children.get(0);
                byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data);
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent
        (TriggerEventType.NODELOST, "mock_trigger_name",
         Collections.singletonList(cloudManager.getTimeSource().getTimeNs()),
         Collections.singletonList(sourceNodeName),
         CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));
  }