Java Code Examples for org.apache.solr.client.solrj.request.CollectionAdminRequest#MoveReplica

public void testScheduledTriggerFailure() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testScheduledTriggerFailure.json");
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 10) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals(0, count);
}
 

public void testUtilizeNodeFailure() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure.json"); //(Map) Utils.fromJSONString(state);
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 100) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals("count = " + count, 0, count);
}
 

public void testUtilizeNodeFailure2() throws Exception {
  @SuppressWarnings({"rawtypes"})
  Map jsonObj = (Map) loadFromResource("testUtilizeNodeFailure2.json");
  SolrCloudManager cloudManager = createCloudManager(jsonObj);
  Suggester suggester = createSuggester(cloudManager, jsonObj, null);
  int count = 0;
  while (count < 100) {
    CollectionAdminRequest.MoveReplica op = (CollectionAdminRequest.MoveReplica) suggester.getSuggestion();
    if (op == null) break;
    count++;
    if (log.isInfoEnabled()) {
      log.info("OP:{}", op.getParams());
    }
    suggester = createSuggester(cloudManager, jsonObj, suggester);
  }

  assertEquals("count = " + count, 1, count);
}
 

@Test
public void testSimulatorFromSnapshot() throws Exception {
  Path tmpPath = createTempDir();
  File tmpDir = tmpPath.toFile();
  SnapshotCloudManager snapshotCloudManager = new SnapshotCloudManager(realManager, null);
  snapshotCloudManager.saveSnapshot(tmpDir, true, false);
  SnapshotCloudManager snapshotCloudManager1 = SnapshotCloudManager.readSnapshot(tmpDir);
  try (SimCloudManager simCloudManager = SimCloudManager.createCluster(snapshotCloudManager1, null, TimeSource.get("simTime:50"))) {
    SimSolrCloudTestCase.assertClusterStateEquals(snapshotCloudManager.getClusterStateProvider().getClusterState(), simCloudManager.getClusterStateProvider().getClusterState());
    assertNodeStateProvider(snapshotCloudManager, simCloudManager, "freedisk");
    assertDistribStateManager(snapshotCloudManager.getDistribStateManager(), simCloudManager.getDistribStateManager());
    ClusterState state = simCloudManager.getClusterStateProvider().getClusterState();
    Replica r = state.getCollection(CollectionAdminParams.SYSTEM_COLL).getReplicas().get(0);
    // get another node
    String target = null;
    for (String node : simCloudManager.getClusterStateProvider().getLiveNodes()) {
      if (!node.equals(r.getNodeName())) {
        target = node;
        break;
      }
    }
    if (target == null) {
      fail("can't find suitable target node for replica " + r + ", liveNodes=" + simCloudManager.getClusterStateProvider().getLiveNodes());
    }
    CollectionAdminRequest.MoveReplica moveReplica = CollectionAdminRequest
        .moveReplica(CollectionAdminParams.SYSTEM_COLL, r.getName(), target);
    log.info("################");
    simCloudManager.simGetSolrClient().request(moveReplica);
  }
}
 

@SuppressWarnings({"rawtypes"})
SolrRequest tryEachNode(boolean strict) {
  //iterate through elements and identify the least loaded
  List<Violation> leastSeriousViolation = null;
  Row bestSrcRow = null;
  Row bestTargetRow = null;
  ReplicaInfo sourceReplicaInfo = null;
  List<Pair<ReplicaInfo, Row>> validReplicas = getValidReplicas(true, true, -1);
  validReplicas.sort(leaderLast);
  for (int i1 = 0; i1 < validReplicas.size(); i1++) {
    lastBestDeviation = null;
    Pair<ReplicaInfo, Row> fromReplica = validReplicas.get(i1);
    Row fromRow = fromReplica.second();
    ReplicaInfo ri = fromReplica.first();
    if (ri == null) continue;
    final int i = session.indexOf(fromRow.node);
    int stopAt = force ? 0 : i;
    Row targetRow = null;
    for (int j = session.matrix.size() - 1; j >= stopAt; j--) {
      targetRow = session.matrix.get(j);
      if (targetRow.node.equals(fromRow.node)) continue;
      if (!isNodeSuitableForReplicaAddition(targetRow, fromRow)) continue;
      targetRow = targetRow.addReplica(ri.getCollection(), ri.getShard(), ri.getType(), strict); // add replica to target first
      Row srcRowModified = targetRow.session.getNode(fromRow.node).removeReplica(ri.getCollection(), ri.getShard(), ri.getType());//then remove replica from source node
      List<Violation> errs = testChangedMatrix(strict, srcRowModified.session);
      Policy.Session tmpSession = srcRowModified.session;

      if (!containsNewErrors(errs) &&
          isLessSerious(errs, leastSeriousViolation) &&
          (force || (tmpSession.indexOf(srcRowModified.node) < tmpSession.indexOf(targetRow.node)))) {

        int result = -1;
        if (!force && srcRowModified.isLive && targetRow.isLive)  {
          result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), true);
          if (result == 0) result = tmpSession.getPolicy().getClusterPreferences().get(0).compare(srcRowModified, tmpSession.getNode(targetRow.node), false);
        }

        if (result <= 0) {
          leastSeriousViolation = errs;
          bestSrcRow = srcRowModified;
          sourceReplicaInfo = ri;
          bestTargetRow = targetRow;
        }
      }
    }
  }
  if (bestSrcRow != null) {
    this.session = bestSrcRow.session;
    return new CollectionAdminRequest.MoveReplica(
        sourceReplicaInfo.getCollection(),
        sourceReplicaInfo.getName(),
        bestTargetRow.node);
  }
  return null;
}
 

@Test
  public void testMoveReplicaMainCollection() throws Exception {
    String prefix = "testMoveReplicaMainCollection";
    String xyz = prefix + "_xyz";
    String abc = prefix + "_abc";

    CloudSolrClient solrClient = cluster.getSolrClient();

    String setClusterPolicyCommand = "{" +
        " 'set-cluster-policy': [" +
        "      {'cores':'<10', 'node':'#ANY'}," +
        "      {'replica':'<2', 'node':'#ANY'}," +
        "    ]" +
        "}";
    @SuppressWarnings({"rawtypes"})
    SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
    solrClient.request(req);

    String chosenNode = cluster.getRandomJetty(random()).getNodeName();
    log.info("Chosen node {} for collection {}", chosenNode, abc);
    CollectionAdminRequest.createCollection(abc, 1, 1)
        .setCreateNodeSet(chosenNode) // randomize to avoid choosing the first node always
        .process(solrClient);
    CollectionAdminRequest.createCollection(xyz, 1, 1)
        .setWithCollection(abc)
        .process(solrClient);

    String otherNode = null;
    for (JettySolrRunner jettySolrRunner : cluster.getJettySolrRunners()) {
      if (!chosenNode.equals(jettySolrRunner.getNodeName())) {
        otherNode = jettySolrRunner.getNodeName();
      }
    }

    DocCollection collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz);
    DocCollection withCollection = solrClient.getZkStateReader().getClusterState().getCollection(abc);
    assertNull(collection.getReplicas(otherNode)); // sanity check
    assertNull(withCollection.getReplicas(otherNode)); // sanity check

    CollectionAdminRequest.MoveReplica moveReplica = new CollectionAdminRequest.MoveReplica(xyz, collection.getReplicas().iterator().next().getName(), otherNode);
    moveReplica.setWaitForFinalState(true);
    moveReplica.process(solrClient);
//    zkClient().printLayoutToStdOut();
    collection = solrClient.getZkStateReader().getClusterState().getCollection(xyz); // refresh
    DocCollection withCollectionRefreshed = solrClient.getZkStateReader().getClusterState().getCollection(abc); // refresh
    assertTrue(collection.getReplicas().stream().noneMatch(
        replica -> withCollectionRefreshed.getReplicas(replica.getNodeName()) == null
            || withCollectionRefreshed.getReplicas(replica.getNodeName()).isEmpty()));
  }
 

@Test
// 12-Jun-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 17-Mar-2018 This JIRA is fixed, but this test still fails
//17-Aug-2018 commented  @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
// commented out on: 17-Feb-2019   @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // annotated on: 24-Dec-2018
public void testFailedMove() throws Exception {
  String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove;
  int REPLICATION = 2;

  CloudSolrClient cloudClient = cluster.getSolrClient();

  // random create tlog or pull type replicas with nrt
  boolean isTlog = random().nextBoolean();
  CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0);
  create.setAutoAddReplicas(false);
  cloudClient.request(create);

  addDocs(coll, 100);

  NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
  String overseerLeader = (String) overSeerStatus.get("leader");

  // don't kill overseer in this test
  Replica replica;
  int count = 10;
  do {
    replica = getRandomReplica(coll, cloudClient);
  } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0);
  assertNotNull("could not find non-overseer replica???", replica);
  Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
  ArrayList<String> l = new ArrayList<>(liveNodes);
  Collections.shuffle(l, random());
  String targetNode = null;
  for (String node : liveNodes) {
    if (!replica.getNodeName().equals(node) && !overseerLeader.equals(node)) {
      targetNode = node;
      break;
    }
  }
  assertNotNull(targetNode);
  CollectionAdminRequest.MoveReplica moveReplica = createMoveReplicaRequest(coll, replica, targetNode);
  moveReplica.setInPlaceMove(inPlaceMove);
  // start moving
  String asyncId = IdUtils.randomId();
  moveReplica.processAsync(asyncId, cloudClient);
  // shut down target node
  for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
    if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) {
      JettySolrRunner j = cluster.stopJettySolrRunner(i);
      cluster.waitForJettyToStop(j);
      break;
    }
  }
  CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus(asyncId);
  // wait for async request success
  boolean success = true;
  for (int i = 0; i < 200; i++) {
    CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
    assertNotSame(rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED);
    if (rsp.getRequestStatus() == RequestStatusState.FAILED) {
      success = false;
      break;
    }
    Thread.sleep(500);
  }
  assertFalse(success);

  if (log.isInfoEnabled()) {
    log.info("--- current collection state: {}", cloudClient.getZkStateReader().getClusterState().getCollection(coll));
  }
  assertEquals(100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound());
}
 

private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(String coll, Replica replica, String targetNode, String shardId) {
  return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName());
}
 

private CollectionAdminRequest.MoveReplica createMoveReplicaRequest(String coll, Replica replica, String targetNode) {
  return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode);
}
 

@Test
  // commented out on: 24-Dec-2018   @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 28-June-2018
  public void testExecute() throws Exception {
    SolrClient solrClient = cluster.simGetSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 120, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }

    String sourceNodeName = cluster.getSimClusterStateProvider().simGetRandomNode();
    ClusterState clusterState = cluster.getClusterStateProvider().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<String> otherNodes = cluster.getClusterStateProvider().getLiveNodes().stream()
        .filter(node -> !node.equals(sourceNodeName)).collect(Collectors.toList());
    assertFalse(otherNodes.isEmpty());
    String survivor = otherNodes.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(cluster.getLoader(), cluster, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor);
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (cluster.getDistribStateManager().hasData(parentPath)) {
              java.util.List<String> children = cluster.getDistribStateManager().listData(parentPath);
              if (!children.isEmpty()) {
                String child = children.get(0);
                VersionedData data = cluster.getDistribStateManager().getData(parentPath + "/" + child);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data.getData());
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST,
          "mock_trigger_name", Collections.singletonList(SIM_TIME_SOURCE.getTimeNs()),
          Collections.singletonList(sourceNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(cluster, null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    if (log.isInfoEnabled()) {
      log.info("Collection ready after {} ms", CloudUtil.waitForState(cluster, collectionName, 300, TimeUnit.SECONDS,
          CloudUtil.clusterShape(1, 2, false, true)));
    }
  }
 

@Test
  public void testExecute() throws Exception {
    CloudSolrClient solrClient = cluster.getSolrClient();
    String collectionName = "testExecute";
    CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
        "conf", 1, 2);
    create.setMaxShardsPerNode(1);
    create.process(solrClient);
    
    cluster.waitForActiveCollection(collectionName, 1, 2);

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));

    JettySolrRunner sourceNode = cluster.getRandomJetty(random());
    String sourceNodeName = sourceNode.getNodeName();
    ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
    DocCollection docCollection = clusterState.getCollection(collectionName);
    List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
    assertNotNull(replicas);
    assertFalse(replicas.isEmpty());

    List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
        .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
    assertFalse(otherJetties.isEmpty());
    JettySolrRunner survivor = otherJetties.get(0);

    try (ExecutePlanAction action = new ExecutePlanAction()) {
      action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan"));

      // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
      AtomicBoolean znodeCreated = new AtomicBoolean(false);

      CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
      CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
        @Override
        public void setAsyncId(String asyncId) {
          super.setAsyncId(asyncId);
          String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
          try {
            if (zkClient().exists(parentPath, true)) {
              java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
              if (!children.isEmpty()) {
                String child = children.get(0);
                byte[] data = zkClient().getData(parentPath + "/" + child, null, null, true);
                @SuppressWarnings({"rawtypes"})
                Map m = (Map) Utils.fromJSON(data);
                if (m.containsKey("requestid")) {
                  znodeCreated.set(m.get("requestid").equals(asyncId));
                }
              }
            }
          } catch (Exception e) {
            throw new RuntimeException(e);
          }

        }
      };
      List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
      NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent
        (TriggerEventType.NODELOST, "mock_trigger_name",
         Collections.singletonList(cloudManager.getTimeSource().getTimeNs()),
         Collections.singletonList(sourceNodeName),
         CollectionParams.CollectionAction.MOVEREPLICA.toLower());
      ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
          new HashMap<>(Collections.singletonMap("operations", operations)));
      action.process(nodeLostEvent, actionContext);

//      assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
      @SuppressWarnings({"unchecked"})
      List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
      assertNotNull(responses);
      assertEquals(2, responses.size());
      NamedList<Object> response = responses.get(0);
      assertNull(response.get("failure"));
      assertNotNull(response.get("success"));
    }

    waitForState("Timed out waiting for replicas of new collection to be active",
        collectionName, clusterShape(1, 2));
  }