Java Code Examples for org.apache.solr.common.cloud.DocCollection#getSlicesMap()

protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
    throws KeeperException, InterruptedException {

    zkStateReader.forceUpdateCollection(collection);
    ClusterState clusterState = zkStateReader.getClusterState();
    final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
    if (docCollection == null || docCollection.getSlices() == null) {
      throw new IllegalArgumentException("Cannot find collection:" + collection);
    }

    Map<String,Slice> slices = docCollection.getSlicesMap();
    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
      Slice slice = entry.getValue();
      if (slice.getState() != Slice.State.ACTIVE) {
        fail("Not all shards are ACTIVE - found a shard " + slice.getName() + " that is: " + slice.getState());
      }
      Map<String,Replica> shards = slice.getReplicasMap();
      for (Map.Entry<String,Replica> shard : shards.entrySet()) {
        Replica replica = shard.getValue();
        if (replica.getState() != Replica.State.ACTIVE) {
          fail("Not all replicas are ACTIVE - found a replica " + replica.getName() + " that is: " + replica.getState());
        }
      }
    }
}
 

private void waitForCoreNodeName(CoreDescriptor descriptor) {
  int retryCount = 320;
  log.debug("look for our core node name");
  while (retryCount-- > 0) {
    final DocCollection docCollection = zkStateReader.getClusterState()
        .getCollectionOrNull(descriptor.getCloudDescriptor().getCollectionName());
    if (docCollection != null && docCollection.getSlicesMap() != null) {
      final Map<String, Slice> slicesMap = docCollection.getSlicesMap();
      for (Slice slice : slicesMap.values()) {
        for (Replica replica : slice.getReplicas()) {
          // TODO: for really large clusters, we could 'index' on this

          String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
          String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

          String msgNodeName = getNodeName();
          String msgCore = descriptor.getName();

          if (msgNodeName.equals(nodeName) && core.equals(msgCore)) {
            descriptor.getCloudDescriptor()
                .setCoreNodeName(replica.getName());
            getCoreContainer().getCoresLocator().persist(getCoreContainer(), descriptor);
            return;
          }
        }
      }
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }
}
 

String waitForCoreNodeName(String collectionName, String msgNodeName, String msgCore) {
  int retryCount = 320;
  while (retryCount-- > 0) {
    final DocCollection docCollection = zkStateReader.getClusterState().getCollectionOrNull(collectionName);
    if (docCollection != null && docCollection.getSlicesMap() != null) {
      Map<String,Slice> slicesMap = docCollection.getSlicesMap();
      for (Slice slice : slicesMap.values()) {
        for (Replica replica : slice.getReplicas()) {
          // TODO: for really large clusters, we could 'index' on this

          String nodeName = replica.getStr(ZkStateReader.NODE_NAME_PROP);
          String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);

          if (nodeName.equals(msgNodeName) && core.equals(msgCore)) {
            return replica.getName();
          }
        }
      }
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }
  throw new SolrException(ErrorCode.SERVER_ERROR, "Could not find coreNodeName");
}
 

public ZkWriteCommand updateShardState(ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  log.info("Update shard state invoked for collection: {} with message: {}", collectionName, message);

  DocCollection collection = clusterState.getCollection(collectionName);
  Map<String, Slice> slicesCopy = new LinkedHashMap<>(collection.getSlicesMap());
  for (String key : message.keySet()) {
    if (ZkStateReader.COLLECTION_PROP.equals(key)) continue;
    if (Overseer.QUEUE_OPERATION.equals(key)) continue;

    Slice slice = collection.getSlice(key);
    if (slice == null) {
      throw new RuntimeException("Overseer.updateShardState unknown collection: " + collectionName + " slice: " + key);
    }
    if (log.isInfoEnabled()) {
      log.info("Update shard state {} to {}", key, message.getStr(key));
    }
    Map<String, Object> props = slice.shallowCopy();
    
    if (Slice.State.getState(message.getStr(key)) == Slice.State.ACTIVE) {
      props.remove(Slice.PARENT);
      props.remove("shard_parent_node");
      props.remove("shard_parent_zk_session");
    }
    props.put(ZkStateReader.STATE_PROP, message.getStr(key));
    // we need to use epoch time so that it's comparable across Overseer restarts
    props.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs()));
    Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props,collectionName);
    slicesCopy.put(slice.getName(), newSlice);
  }

  return new ZkWriteCommand(collectionName, collection.copyWithSlices(slicesCopy));
}
 

public ZkWriteCommand deleteShard(final ClusterState clusterState, ZkNodeProps message) {
  final String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP);
  final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;

  log.info("Removing collection: {} shard: {}  from clusterstate", collection, sliceId);

  DocCollection coll = clusterState.getCollection(collection);

  Map<String, Slice> newSlices = new LinkedHashMap<>(coll.getSlicesMap());
  newSlices.remove(sliceId);

  DocCollection newCollection = coll.copyWithSlices(newSlices);
  return new ZkWriteCommand(collection, newCollection);
}
 

private String getShardId(DocCollection collection, String coreNodeName) {
  if (collection == null) return null;
  Map<String,Slice> slices = collection.getSlicesMap();
  if (slices != null) {
    for (Slice slice : slices.values()) {
      for (Replica replica : slice.getReplicas()) {
        String cnn = replica.getName();
        if (coreNodeName.equals(cnn)) {
          return slice.getName();
        }
      }
    }
  }
  return null;
}
 

/**
 * See if coreNodeName has been taken over by another baseUrl and unload core
 * + throw exception if it has been.
 */
public static void checkSharedFSFailoverReplaced(CoreContainer cc, CoreDescriptor desc) {
  if (!cc.isSharedFs(desc)) return;

  ZkController zkController = cc.getZkController();
  String thisCnn = zkController.getCoreNodeName(desc);
  String thisBaseUrl = zkController.getBaseUrl();

  log.debug("checkSharedFSFailoverReplaced running for coreNodeName={} baseUrl={}", thisCnn, thisBaseUrl);

  // if we see our core node name on a different base url, unload
  final DocCollection docCollection = zkController.getClusterState().getCollectionOrNull(desc.getCloudDescriptor().getCollectionName());
  if (docCollection != null && docCollection.getSlicesMap() != null) {
    Map<String,Slice> slicesMap = docCollection.getSlicesMap();
    for (Slice slice : slicesMap.values()) {
      for (Replica replica : slice.getReplicas()) {

        String cnn = replica.getName();
        String baseUrl = replica.getStr(ZkStateReader.BASE_URL_PROP);
        log.debug("compare against coreNodeName={} baseUrl={}", cnn, baseUrl);

        if (thisCnn != null && thisCnn.equals(cnn)
            && !thisBaseUrl.equals(baseUrl)) {
          if (cc.getLoadedCoreNames().contains(desc.getName())) {
            cc.unload(desc.getName());
          }

          try {
            FileUtils.deleteDirectory(desc.getInstanceDir().toFile());
          } catch (IOException e) {
            SolrException.log(log, "Failed to delete instance dir for core:"
                + desc.getName() + " dir:" + desc.getInstanceDir());
          }
          log.error("{}",
              new SolrException(ErrorCode.SERVER_ERROR, "Will not load SolrCore " + desc.getName()
                  + " because it has been replaced due to failover.")); // logOk
          throw new SolrException(ErrorCode.SERVER_ERROR,
              "Will not load SolrCore " + desc.getName()
                  + " because it has been replaced due to failover.");
        }
      }
    }
  }
}
 

public List<ZkWriteCommand> downNode(ClusterState clusterState, ZkNodeProps message) {
  List<ZkWriteCommand> zkWriteCommands = new ArrayList<>();
  String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);

  log.debug("DownNode state invoked for node: {}", nodeName);

  Map<String, DocCollection> collections = clusterState.getCollectionsMap();
  for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
    String collection = entry.getKey();
    DocCollection docCollection = entry.getValue();

    Map<String,Slice> slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap());

    boolean needToUpdateCollection = false;
    for (Entry<String, Slice> sliceEntry : slicesCopy.entrySet()) {
      Slice slice = sliceEntry.getValue();
      Map<String, Replica> newReplicas = slice.getReplicasCopy();

      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        String rNodeName = replica.getNodeName();
        if (rNodeName == null) {
          throw new RuntimeException("Replica without node name! " + replica);
        }
        if (rNodeName.equals(nodeName)) {
          log.debug("Update replica state for {} to {}", replica, Replica.State.DOWN);
          Map<String, Object> props = replica.shallowCopy();
          props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
          Replica newReplica = new Replica(replica.getName(), props, collection, slice.getName());
          newReplicas.put(replica.getName(), newReplica);
          needToUpdateCollection = true;
        }
      }

      Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy(),collection);
      slicesCopy.put(slice.getName(), newSlice);
    }

    if (needToUpdateCollection) {
      zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy)));
    }
  }

  return zkWriteCommands;
}
 

@Test
public void testCoreRegistration() throws Exception {
  System.setProperty("solrcloud.update.delay", "1");

  assertEquals(0, CollectionAdminRequest.createCollection("testcore", "conf", 1, 1)
      .setCreateNodeSet(cluster.getJettySolrRunner(0).getNodeName())
      .process(cluster.getSolrClient()).getStatus());

  ZkController zkController2 = cluster.getJettySolrRunner(1).getCoreContainer().getZkController();

  String host = zkController2.getHostName();
  
  // slight pause - TODO: takes an oddly long amount of time to schedule tasks
  // with almost no delay ...
  ClusterState clusterState2 = null;
  Map<String,Slice> slices = null;
  for (int i = 75; i > 0; i--) {
    clusterState2 = zkController2.getClusterState();
    DocCollection docCollection = clusterState2.getCollectionOrNull("testcore");
    slices = docCollection == null ? null : docCollection.getSlicesMap();
    
    if (slices != null && slices.containsKey("shard1")
        && slices.get("shard1").getReplicasMap().size() > 0) {
      break;
    }
    Thread.sleep(500);
  }

  assertNotNull(slices);
  assertTrue(slices.containsKey("shard1"));

  Slice slice = slices.get("shard1");
  assertEquals("shard1", slice.getName());

  Map<String,Replica> shards = slice.getReplicasMap();

  assertEquals(1, shards.size());

  // assert this is core of container1
  Replica zkProps = shards.values().iterator().next();

  assertNotNull(zkProps);

  assertEquals(host + ":" +cluster.getJettySolrRunner(0).getLocalPort()+"_solr", zkProps.getStr(ZkStateReader.NODE_NAME_PROP));

  assertTrue(zkProps.getStr(ZkStateReader.BASE_URL_PROP).contains("http://" + host + ":"+cluster.getJettySolrRunner(0).getLocalPort()+"/solr")
    || zkProps.getStr(ZkStateReader.BASE_URL_PROP).contains("https://" + host + ":"+cluster.getJettySolrRunner(0).getLocalPort()+"/solr") );

  // assert there are 3 live nodes
  Set<String> liveNodes = clusterState2.getLiveNodes();
  assertNotNull(liveNodes);
  assertEquals(3, liveNodes.size());

  // shut down node 2
  JettySolrRunner j = cluster.stopJettySolrRunner(2);

  // slight pause (15s timeout) for watch to trigger
  for(int i = 0; i < (5 * 15); i++) {
    if(zkController2.getClusterState().getLiveNodes().size() == 2) {
      break;
    }
    Thread.sleep(200);
  }
  
  cluster.waitForJettyToStop(j);

  assertEquals(2, zkController2.getClusterState().getLiveNodes().size());

  cluster.getJettySolrRunner(1).stop();
  cluster.getJettySolrRunner(1).start();
  
  // pause for watch to trigger
  for(int i = 0; i < 200; i++) {
    if (cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getClusterState().liveNodesContain(
        cluster.getJettySolrRunner(1).getCoreContainer().getZkController().getNodeName())) {
      break;
    }
    Thread.sleep(100);
  }

  assertTrue(cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getClusterState().liveNodesContain(
      cluster.getJettySolrRunner(1).getCoreContainer().getZkController().getNodeName()));

  // core.close();  // don't close - this core is managed by container1 now
}