Java Code Examples for org.apache.solr.util.RTimer

The following examples show how to use org.apache.solr.util.RTimer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: IndexFingerprint.java    License: Apache License 2.0 6 votes vote down vote up
/** Opens a new realtime searcher and returns it's (possibly cached) fingerprint */
public static IndexFingerprint getFingerprint(SolrCore core, long maxVersion) throws IOException {
  RTimer timer = new RTimer();
  core.getUpdateHandler().getUpdateLog().openRealtimeSearcher();
  RefCounted<SolrIndexSearcher> newestSearcher = core.getUpdateHandler().getUpdateLog().uhandler.core.getRealtimeSearcher();
  try {
    IndexFingerprint f = newestSearcher.get().getIndexFingerprint(maxVersion);
    final double duration = timer.stop();
    log.info("IndexFingerprint millis:{} result:{}",duration, f);
    return f;
  } finally {
    if (newestSearcher != null) {
      newestSearcher.decref();
    }
  }
}
 
Example 2
Source Project: lucene-solr   Source File: FacetRequest.java    License: Apache License 2.0 6 votes vote down vote up
/** Process the request with the facet context settings, a parameter-object. */
final Object process(FacetContext fcontext) throws IOException {
  @SuppressWarnings("rawtypes")
  FacetProcessor facetProcessor = createFacetProcessor(fcontext);

  FacetDebugInfo debugInfo = fcontext.getDebugInfo();
  if (debugInfo == null) {
    facetProcessor.process();
  } else {
    if (fcontext.filter != null) {
      debugInfo.setFilter(fcontext.filter.toString());
    }
    debugInfo.setReqDescription(getFacetDescription());
    debugInfo.setProcessor(facetProcessor.getClass().getSimpleName());
    debugInfo.putInfoItem("domainSize", (long) fcontext.base.size());
    RTimer timer = new RTimer();
    try {
      facetProcessor.process();
    }finally {
      debugInfo.setElapse((long) timer.getTime());
    }
  }

  return facetProcessor.getResponse(); 
}
 
Example 3
Source Project: lucene-solr   Source File: TestSearchPerf.java    License: Apache License 2.0 6 votes vote down vote up
int doSetGen(int iter, Query q) throws Exception {
  SolrQueryRequest req = lrf.makeRequest();

  SolrIndexSearcher searcher = req.getSearcher();

  final RTimer timer = new RTimer();

  int ret = 0;
  for (int i=0; i<iter; i++) {
    DocSet set = searcher.getDocSetNC(q, null);
    ret += set.size();
  }

  double elapsed = timer.getTime();
  System.out.println("ret="+ret+ " time="+elapsed+" throughput="+iter*1000/(elapsed+1));

  req.close();
  assertTrue(ret>0);  // make sure we did some work
  return ret;
}
 
Example 4
Source Project: lucene-solr   Source File: TestSearchPerf.java    License: Apache License 2.0 6 votes vote down vote up
int doListGen(int iter, Query q, List<Query> filt, boolean cacheQuery, boolean cacheFilt) throws Exception {
  SolrQueryRequest req = lrf.makeRequest();

  SolrIndexSearcher searcher = req.getSearcher();

  final RTimer timer = new RTimer();

  int ret = 0;
  for (int i=0; i<iter; i++) {
    DocList l = searcher.getDocList(q, filt, (Sort)null, 0, 10, (cacheQuery?0:SolrIndexSearcher.NO_CHECK_QCACHE)|(cacheFilt?0:SolrIndexSearcher.NO_CHECK_FILTERCACHE) );
    ret += l.matches();
  }

  double elapsed = timer.getTime();
  System.out.println("ret="+ret+ " time="+elapsed+" throughput="+iter*1000/(elapsed+1));

  req.close();
  assertTrue(ret>0);  // make sure we did some work
  return ret;
}
 
Example 5
Source Project: lucene-solr   Source File: TestSolrJ.java    License: Apache License 2.0 6 votes vote down vote up
public void doCommitPerf() throws Exception {

    try (HttpSolrClient client = getHttpSolrClient("http://127.0.0.1:8983/solr")) {

      final RTimer timer = new RTimer();

      for (int i = 0; i < 10000; i++) {
        SolrInputDocument doc = new SolrInputDocument();
        doc.addField("id", Integer.toString(i % 13));
        client.add(doc);
        client.commit(true, true, true);
      }

      System.out.println("TIME: " + timer.getTime());
    }

  }
 
Example 6
Source Project: lucene-solr   Source File: TestBlobHandler.java    License: Apache License 2.0 6 votes vote down vote up
public static void postAndCheck(CloudSolrClient cloudClient, String baseUrl, String blobName, ByteBuffer bytes, int count) throws Exception {
  postData(cloudClient, baseUrl, blobName, bytes);

  String url;
  MapWriter map = null;
  final RTimer timer = new RTimer();
  int i = 0;
  for (; i < 150; i++) {//15 secs
    url = baseUrl + "/.system/blob/" + blobName;
    map = TestSolrConfigHandlerConcurrent.getAsMap(url, cloudClient);
    String numFound = map._getStr(asList("response", "numFound"),null);
    if (!("" + count).equals(numFound)) {
      Thread.sleep(100);
      continue;
    }

    assertEquals("" + bytes.limit(), map._getStr("response/docs[0]/size",null));
    return;
  }
  fail(StrUtils.formatString("Could not successfully add blob after {0} attempts. Expecting {1} items. time elapsed {2}  output  for url is {3}",
      i, count, timer.getTime(), map.toString()));
}
 
Example 7
Source Project: lucene-solr   Source File: HttpPartitionOnCommitTest.java    License: Apache License 2.0 6 votes vote down vote up
protected void sendCommitWithRetry(Replica replica) throws Exception {
  String replicaCoreUrl = replica.getCoreUrl();
  log.info("Sending commit request to: {}", replicaCoreUrl);
  final RTimer timer = new RTimer();
  try (HttpSolrClient client = getHttpSolrClient(replicaCoreUrl)) {
    try {
      client.commit();

      if (log.isInfoEnabled()) {
        log.info("Sent commit request to {} OK, took {}ms", replicaCoreUrl, timer.getTime());
      }
    } catch (Exception exc) {
      Throwable rootCause = SolrException.getRootCause(exc);
      if (rootCause instanceof NoHttpResponseException) {
        log.warn("No HTTP response from sending commit request to {}; will re-try after waiting 3 seconds", replicaCoreUrl);
        Thread.sleep(3000);
        client.commit();
        log.info("Second attempt at sending commit to {} succeeded", replicaCoreUrl);
      } else {
        throw exc;
      }
    }
  }
}
 
Example 8
Source Project: lucene-solr   Source File: SolrExampleTests.java    License: Apache License 2.0 5 votes vote down vote up
@Test
@Monster("Only useful to verify the performance of serialization+ deserialization")
// ant -Dtestcase=SolrExampleBinaryTest -Dtests.method=testQueryPerf -Dtests.monster=true test
public void testQueryPerf() throws Exception {
  HttpSolrClient client = (HttpSolrClient) getSolrClient();
  client.deleteByQuery("*:*");
  client.commit();
  ArrayList<SolrInputDocument> docs = new ArrayList<>();
  int id = 0;
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "apple", "cat", "a", "inStock", true, "popularity", 12, "price", .017));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "lg", "cat", "a", "inStock", false, "popularity", 13, "price", 16.04));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "samsung", "cat", "a", "inStock", true, "popularity", 14, "price", 12.34));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "lg", "cat", "b", "inStock", false, "popularity", 24, "price", 51.39));
  docs.add(makeTestDoc("id", id++, "features", "aaa", "manu", "nokia", "cat", "b", "inStock", true, "popularity", 28, "price", 131.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "a", "inStock", false, "popularity", 32));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "htc", "cat", "a", "inStock", true, "popularity", 31, "price", 131.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "apple", "cat", "b", "inStock", false, "popularity", 36));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "lg", "cat", "b", "inStock", true, "popularity", 37, "price", 1.39));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "b", "inStock", false, "popularity", 38, "price", 47.98));
  docs.add(makeTestDoc("id", id++, "features", "bbb", "manu", "ztc", "cat", "b", "inStock", true, "popularity", -38));
  docs.add(makeTestDoc("id", id++, "cat", "b")); // something not matching all fields
  client.add(docs);
  client.commit();
  //this sets the cache
  QueryResponse rsp = getSolrClient().query(new SolrQuery("*:*").setRows(20));

  RTimer timer = new RTimer();
  int count = 10000;
  log.info("Started perf test....");
  for(int i=0;i< count; i++){
    rsp = getSolrClient().query(new SolrQuery("*:*").setRows(20));
  }

  if (log.isInfoEnabled()) {
    log.info("time taken to execute {} queries is {} ms", count, timer.getTime());
  }

}
 
Example 9
Source Project: lucene-solr   Source File: UpdateLog.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Used to seed all version buckets with the max value of the version field in the index.
 */
protected Long seedBucketsWithHighestVersion(SolrIndexSearcher newSearcher, VersionInfo versions) {
  Long highestVersion = null;
  final RTimer timer = new RTimer();

  try (RecentUpdates recentUpdates = getRecentUpdates()) {
    long maxVersionFromRecent = recentUpdates.getMaxRecentVersion();
    long maxVersionFromIndex = versions.getMaxVersionFromIndex(newSearcher);

    long maxVersion = Math.max(maxVersionFromIndex, maxVersionFromRecent);
    if (maxVersion == 0L) {
      maxVersion = versions.getNewClock();
      log.info("Could not find max version in index or recent updates, using new clock {}", maxVersion);
    }

    // seed all version buckets with the highest value from recent and index
    versions.seedBucketsWithHighestVersion(maxVersion);

    highestVersion = maxVersion;
  } catch (IOException ioExc) {
    log.warn("Failed to determine the max value of the version field due to: ", ioExc);
  }

  if (debug) {
    log.debug("Took {}ms to seed version buckets with highest version {}",
        timer.getTime(), highestVersion);
  }

  return highestVersion;
}
 
Example 10
Source Project: lucene-solr   Source File: SystemInfoHandler.java    License: Apache License 2.0 5 votes vote down vote up
private void initHostname() {
  if (null != System.getProperty(PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP, null)) {
    log.info("Resolving canonical hostname for local host prevented due to '{}' sysprop",
             PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP);
    hostname = null;
    return;
  }
  
  RTimer timer = new RTimer();
  try {
    InetAddress addr = InetAddress.getLocalHost();
    hostname = addr.getCanonicalHostName();
  } catch (Exception e) {
    log.warn("Unable to resolve canonical hostname for local host, possible DNS misconfiguration. SET THE '{}' {}"
        , PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP
        , " sysprop to true on startup to prevent future lookups if DNS can not be fixed.", e);
    hostname = null;
    return;
  }
  timer.stop();
  
  if (15000D < timer.getTime()) {
    String readableTime = String.format(Locale.ROOT, "%.3f", (timer.getTime() / 1000));
    log.warn("Resolving canonical hostname for local host took {} seconds, possible DNS misconfiguration. Set the '{}' {}"
        , readableTime, PREVENT_REVERSE_DNS_OF_LOCALHOST_SYSPROP,
        " sysprop to true on startup to prevent future lookups if DNS can not be fixed.");
  }
}
 
Example 11
Source Project: lucene-solr   Source File: SolrConfigHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Boolean call() throws Exception {
  final RTimer timer = new RTimer();
  int attempts = 0;
  try (HttpSolrClient solr = new HttpSolrClient.Builder(coreUrl).build()) {
    // eventually, this loop will get killed by the ExecutorService's timeout
    while (true) {
      try {
        long timeElapsed = (long) timer.getTime() / 1000;
        if (timeElapsed >= maxWait) {
          return false;
        }
        log.info("Time elapsed : {} secs, maxWait {}", timeElapsed, maxWait);
        Thread.sleep(100);
        NamedList<Object> resp = solr.httpUriRequest(this).future.get();
        if (resp != null) {
          @SuppressWarnings({"rawtypes"})
          Map m = (Map) resp.get(ZNODEVER);
          if (m != null) {
            remoteVersion = (Number) m.get(prop);
            if (remoteVersion != null && remoteVersion.intValue() >= expectedZkVersion) break;
          }
        }

        attempts++;
        if (log.isInfoEnabled()) {
          log.info(formatString("Could not get expectedVersion {0} from {1} for prop {2}   after {3} attempts", expectedZkVersion, coreUrl, prop, attempts));
        }
      } catch (Exception e) {
        if (e instanceof InterruptedException) {
          break; // stop looping
        } else {
          log.warn("Failed to get /schema/zkversion from {} due to: ", coreUrl, e);
        }
      }
    }
  }
  return true;
}
 
Example 12
ClusterState waitForNewShard(String collectionName, String sliceName) throws KeeperException, InterruptedException {
  log.debug("Waiting for slice {} of collection {} to be available", sliceName, collectionName);
  RTimer timer = new RTimer();
  int retryCount = 320;
  while (retryCount-- > 0) {
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection = clusterState.getCollection(collectionName);

    if (collection == null) {
      throw new SolrException(ErrorCode.SERVER_ERROR,
          "Unable to find collection: " + collectionName + " in clusterstate");
    }
    Slice slice = collection.getSlice(sliceName);
    if (slice != null) {
      if (log.isDebugEnabled()) {
        log.debug("Waited for {}ms for slice {} of collection {} to be available",
            timer.getTime(), sliceName, collectionName);
      }
      return clusterState;
    }
    Thread.sleep(1000);
  }
  throw new SolrException(ErrorCode.SERVER_ERROR,
      "Could not find new slice " + sliceName + " in collection " + collectionName
          + " even after waiting for " + timer.getTime() + "ms"
  );
}
 
Example 13
Source Project: lucene-solr   Source File: TestJavaBinCodec.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({"unchecked"})
public static void doDecodePerf(String[] args) throws Exception {
  int arg=0;
  int nThreads = Integer.parseInt(args[arg++]);
  int nBuffers = Integer.parseInt(args[arg++]);
  final long iter = Long.parseLong(args[arg++]);
  int cacheSz = Integer.parseInt(args[arg++]);

  Random r = new Random(0);

  final byte[][] buffers = new byte[nBuffers][];

  for (int bufnum=0; bufnum<nBuffers; bufnum++) {
    SolrDocument sdoc = new SolrDocument();
    sdoc.put("id", "my_id_" + bufnum);
    sdoc.put("author", str(r, 10 + r.nextInt(10)));
    sdoc.put("address", str(r, 20 + r.nextInt(20)));
    sdoc.put("license", str(r, 10));
    sdoc.put("title", str(r, 5 + r.nextInt(10)));
    sdoc.put("modified_dt", r.nextInt(1000000));
    sdoc.put("creation_dt", r.nextInt(1000000));
    sdoc.put("birthdate_dt", r.nextInt(1000000));
    sdoc.put("clean", r.nextBoolean());
    sdoc.put("dirty", r.nextBoolean());
    sdoc.put("employed", r.nextBoolean());
    sdoc.put("priority", r.nextInt(100));
    sdoc.put("dependents", r.nextInt(6));
    sdoc.put("level", r.nextInt(101));
    sdoc.put("education_level", r.nextInt(10));
    // higher level of reuse for string values
    sdoc.put("state", "S"+r.nextInt(50));
    sdoc.put("country", "Country"+r.nextInt(20));
    sdoc.put("some_boolean", ""+r.nextBoolean());
    sdoc.put("another_boolean", ""+r.nextBoolean());

    buffers[bufnum] = getBytes(sdoc);
  }

  int ret = 0;
  final RTimer timer = new RTimer();
  @SuppressWarnings({"rawtypes"})
  ConcurrentLRUCache underlyingCache = cacheSz > 0 ? new ConcurrentLRUCache<>(cacheSz,cacheSz-cacheSz/10,cacheSz,cacheSz/10,false,true,null) : null;  // the cache in the first version of the patch was 10000,9000,10000,1000,false,true,null
  final JavaBinCodec.StringCache stringCache = underlyingCache==null ? null : new JavaBinCodec.StringCache(underlyingCache);
  if (nThreads <= 0) {
    ret += doDecode(buffers, iter, stringCache);
  } else {
    runInThreads(nThreads, () -> {
      try {
        doDecode(buffers, iter, stringCache);
      } catch (IOException e) {
        e.printStackTrace();
      }
    });
  }

  long n = iter * Math.max(1,nThreads);
  System.out.println("ret=" + ret + " THROUGHPUT=" + (n*1000 / timer.getTime()));
  if (underlyingCache != null) System.out.println("cache: hits=" + underlyingCache.getStats().getCumulativeHits() + " lookups=" + underlyingCache.getStats().getCumulativeLookups() + " size=" + underlyingCache.getStats().getCurrentSize());
}
 
Example 14
Source Project: lucene-solr   Source File: ChaosMonkey.java    License: Apache License 2.0 4 votes vote down vote up
public void startTheMonkey(boolean killLeaders, final int roundPauseUpperLimit) {
  if (!MONKEY_ENABLED) {
    monkeyLog("The Monkey is disabled and will not start");
    return;
  }
  monkeyLog("starting");
  
  
  if (chaosRandom.nextBoolean()) {
    monkeyLog("Jetty will not commit on close");
    TestInjection.skipIndexWriterCommitOnClose = true;
  }

  this.aggressivelyKillLeaders = killLeaders;
  runTimer = new RTimer();
  // TODO: when kill leaders is on, lets kill a higher percentage of leaders
  
  stop = false;
  monkeyThread = new Thread() {

    @Override
    public void run() {
      while (!stop) {
        try {
  
          Thread.sleep(chaosRandom.nextInt(roundPauseUpperLimit));

          causeSomeChaos();
          
        } catch (InterruptedException e) {
          //
        } catch (Exception e) {
          // TODO Auto-generated catch block
          e.printStackTrace();
        }
      }
      monkeyLog("finished");
      monkeyLog("I ran for " + runTimer.getTime() / 1000 + "s. I stopped " + stops + " and I started " + starts
          + ". I also expired " + expires.get() + " and caused " + connloss
          + " connection losses");
    }
  };
  monkeyThread.start();
}
 
Example 15
Source Project: lucene-solr   Source File: MailEntityProcessor.java    License: Apache License 2.0 4 votes vote down vote up
public MessageIterator(Folder folder, int batchSize) {
  super();
  
  try {
    this.folder = folder;
    this.batchSize = batchSize;
    SearchTerm st = getSearchTerm();
    
    log.info("SearchTerm={}", st);
    
    if (st != null || folder instanceof GmailFolder) {
      doBatching = false;
      // Searching can still take a while even though we're only pulling
      // envelopes; unless you're using gmail server-side filter, which is
      // fast
      if (log.isInfoEnabled()) {
        log.info("Searching folder {} for messages", folder.getName());
      }
      final RTimer searchTimer = new RTimer();

      // If using GMail, speed up the envelope processing by doing a
      // server-side
      // search for messages occurring on or after the fetch date (at
      // midnight),
      // which reduces the number of envelopes we need to pull from the
      // server
      // to apply the precise DateTerm filter; GMail server-side search has
      // date
      // granularity only but the local filters are also applied
                
      if (folder instanceof GmailFolder && fetchMailsSince != null) {
        String afterCrit = "after:" + afterFmt.format(fetchMailsSince);
        log.info("Added server-side gmail filter: {}", afterCrit);
        Message[] afterMessages = folder.search(new GmailRawSearchTerm(
            afterCrit));

        if (log.isInfoEnabled()) {
          log.info("GMail server-side filter found {} messages received {} in folder {}"
              , afterMessages.length, afterCrit, folder.getName());
        }
        
        // now pass in the server-side filtered messages to the local filter
        messagesInCurBatch = folder.search((st != null ? st : this), afterMessages);
      } else {
        messagesInCurBatch = folder.search(st);
      }          
      totalInFolder = messagesInCurBatch.length;
      folder.fetch(messagesInCurBatch, fp);
      current = 0;
      if (log.isInfoEnabled()) {
        log.info("Total messages : {}", totalInFolder);
        log.info("Search criteria applied. Batching disabled. Took {} (ms)", searchTimer.getTime()); // logOk
      }
    } else {
      totalInFolder = folder.getMessageCount();
      log.info("Total messages : {}", totalInFolder);
      getNextBatch(batchSize, folder);
    }
  } catch (MessagingException e) {
    throw new DataImportHandlerException(DataImportHandlerException.SEVERE,
        "Message retreival failed", e);
  }
}
 
Example 16
Source Project: lucene-solr   Source File: JoinQuery.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  if (filter == null) {
    boolean debug = rb != null && rb.isDebug();
    RTimer timer = (debug ? new RTimer() : null);
    resultSet = getDocSet();
    if (timer != null) timer.stop();

    if (debug) {
      SimpleOrderedMap<Object> dbg = new SimpleOrderedMap<>();
      dbg.add("time", (long) timer.getTime());
      dbg.add("fromSetSize", fromSetSize);  // the input
      dbg.add("toSetSize", resultSet.size());    // the output

      dbg.add("fromTermCount", fromTermCount);
      dbg.add("fromTermTotalDf", fromTermTotalDf);
      dbg.add("fromTermDirectCount", fromTermDirectCount);
      dbg.add("fromTermHits", fromTermHits);
      dbg.add("fromTermHitsTotalDf", fromTermHitsTotalDf);
      dbg.add("toTermHits", toTermHits);
      dbg.add("toTermHitsTotalDf", toTermHitsTotalDf);
      dbg.add("toTermDirectCount", toTermDirectCount);
      dbg.add("smallSetsDeferred", smallSetsDeferred);
      dbg.add("toSetDocsAdded", resultListDocs);

      // TODO: perhaps synchronize  addDebug in the future...
      rb.addDebug(dbg, "join", JoinQuery.this.toString());
    }

    filter = resultSet.getTopFilter();
  }

  // Although this set only includes live docs, other filters can be pushed down to queries.
  DocIdSet readerSet = filter.getDocIdSet(context, null);
  if (readerSet == null) {
    return null;
  }
  DocIdSetIterator readerSetIterator = readerSet.iterator();
  if (readerSetIterator == null) {
    return null;
  }
  return new ConstantScoreScorer(this, score(), scoreMode, readerSetIterator);
}
 
Example 17
Source Project: lucene-solr   Source File: SplitOp.java    License: Apache License 2.0 4 votes vote down vote up
static Collection<RangeCount> getHashHistogram(SolrIndexSearcher searcher, String prefixField, DocRouter router, DocCollection collection) throws IOException {
  RTimer timer = new RTimer();
  TreeMap<DocRouter.Range,RangeCount> counts = new TreeMap<>();

  Terms terms = MultiTerms.getTerms(searcher.getIndexReader(), prefixField);
  if (terms == null) {
    return counts.values();
  }

  int numPrefixes = 0;
  int numTriLevel = 0;
  int numCollisions = 0;
  long sumBuckets = 0;

  TermsEnum termsEnum = terms.iterator();
  BytesRef term;
  while ((term = termsEnum.next()) != null) {
    numPrefixes++;

    String termStr = term.utf8ToString();
    int firstSep = termStr.indexOf(CompositeIdRouter.SEPARATOR);
    // truncate to first separator since we don't support multiple levels currently
    // NOTE: this does not currently work for tri-level composite ids since the number of bits allocated to the first ID is 16 for a 2 part id
    // and 8 for a 3 part id!
    if (firstSep != termStr.length()-1 && firstSep > 0) {
      numTriLevel++;
      termStr = termStr.substring(0, firstSep+1);
    }

    DocRouter.Range range = router.getSearchRangeSingle(termStr, null, collection);
    int numDocs = termsEnum.docFreq();
    sumBuckets += numDocs;

    RangeCount rangeCount = new RangeCount(range, numDocs);

    RangeCount prev = counts.put(rangeCount.range, rangeCount);
    if (prev != null) {
      // we hit a hash collision or truncated a prefix to first level, so add the buckets together.
      rangeCount.count += prev.count;
      numCollisions++;
    }
  }

  if (log.isInfoEnabled()) {
    log.info("Split histogram: ms={}, numBuckets={} sumBuckets={} numPrefixes={} numTriLevel={} numCollisions={}"
        , timer.getTime(), counts.size(), sumBuckets, numPrefixes, numTriLevel, numCollisions);
  }

  return counts.values();
}
 
Example 18
Source Project: lucene-solr   Source File: SplitOp.java    License: Apache License 2.0 4 votes vote down vote up
/**
 *   Returns a list of range counts sorted by the range lower bound, using the indexed "id" field (i.e. the terms are full IDs, not just prefixes)
 */
static Collection<RangeCount> getHashHistogramFromId(SolrIndexSearcher searcher, String idField, DocRouter router, DocCollection collection) throws IOException {
  RTimer timer = new RTimer();

  TreeMap<DocRouter.Range, RangeCount> counts = new TreeMap<>();

  Terms terms = MultiTerms.getTerms(searcher.getIndexReader(), idField);
  if (terms == null) {
    return counts.values();
  }

  int numPrefixes = 0;
  int numCollisions = 0;
  long sumBuckets = 0;


  byte sep = (byte) CompositeIdRouter.SEPARATOR.charAt(0);
  TermsEnum termsEnum = terms.iterator();
  BytesRef currPrefix = new BytesRef();  // prefix of the previous "id" term
  int bucketCount = 0; // count of the number of docs in the current bucket

  // We're going to iterate over all terms, so do the minimum amount of work per term.
  // Terms are sorted, so all terms sharing a prefix will be grouped together.  The extra work
  // is really just limited to stepping over all the terms in the id field.
  for (;;) {
    BytesRef term = termsEnum.next();

    // compare to current prefix bucket and see if this new term shares the same prefix
    if (term != null && term.length >= currPrefix.length && currPrefix.length > 0) {
      if (StringHelper.startsWith(term, currPrefix)) {
        bucketCount++;  // use 1 since we are dealing with unique ids
        continue;
      }
    }

    // At this point the prefix did not match, so if we had a bucket we were working on, record it.
    if (currPrefix.length > 0) {
      numPrefixes++;
      sumBuckets += bucketCount;
      String currPrefixStr = currPrefix.utf8ToString();
      DocRouter.Range range = router.getSearchRangeSingle(currPrefixStr, null, collection);

      RangeCount rangeCount = new RangeCount(range, bucketCount);
      bucketCount = 0;

      RangeCount prev = counts.put(rangeCount.range, rangeCount);
      if (prev != null) {
        // we hit a hash collision, so add the buckets together.
        rangeCount.count += prev.count;
        numCollisions++;
      }
    }

    // if the current term is null, we ran out of values
    if (term == null) break;

    // find the new prefix (if any)

    // resize if needed
    if (currPrefix.length < term.length) {
      currPrefix.bytes = new byte[term.length+10];
    }

    // Copy the bytes up to and including the separator, and set the length if the separator is found.
    // If there was no separator, then length remains 0 and it's the indicator that we have no prefix bucket
    currPrefix.length = 0;
    for (int i=0; i<term.length; i++) {
      byte b = term.bytes[i + term.offset];
      currPrefix.bytes[i] = b;
      if (b == sep) {
        currPrefix.length = i + 1;
        bucketCount++;
        break;
      }
    }
  }

  if (log.isInfoEnabled()) {
    log.info("Split histogram from idField {}: ms={}, numBuckets={} sumBuckets={} numPrefixes={} numCollisions={}"
        , idField, timer.getTime(), counts.size(), sumBuckets, numPrefixes, numCollisions);
  }

  return counts.values();
}
 
Example 19
Source Project: lucene-solr   Source File: IndexFetcher.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressForbidden(reason = "Need currentTimeMillis for debugging/stats")
private void markReplicationStart() {
  replicationTimer = new RTimer();
  replicationStartTimeStamp = new Date();
}
 
Example 20
Source Project: lucene-solr   Source File: ResponseBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public RTimer getTimer() {
  return timer;
}
 
Example 21
Source Project: lucene-solr   Source File: ResponseBuilder.java    License: Apache License 2.0 4 votes vote down vote up
public void setTimer(RTimer timer) {
  this.timer = timer;
}
 
Example 22
Source Project: lucene-solr   Source File: ZkController.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Ensures that a searcher is registered for the given core and if not, waits until one is registered
 */
private static void ensureRegisteredSearcher(SolrCore core) throws InterruptedException {
  if (!core.getSolrConfig().useColdSearcher) {
    RefCounted<SolrIndexSearcher> registeredSearcher = core.getRegisteredSearcher();
    if (registeredSearcher != null) {
      if (log.isDebugEnabled()) {
        log.debug("Found a registered searcher: {} for core: {}", registeredSearcher.get(), core);
      }
      registeredSearcher.decref();
    } else  {
      @SuppressWarnings({"rawtypes"})
      Future[] waitSearcher = new Future[1];
      if (log.isInfoEnabled()) {
        log.info("No registered searcher found for core: {}, waiting until a searcher is registered before publishing as active", core.getName());
      }
      final RTimer timer = new RTimer();
      RefCounted<SolrIndexSearcher> searcher = null;
      try {
        searcher = core.getSearcher(false, true, waitSearcher, true);
        boolean success = true;
        if (waitSearcher[0] != null)  {
          if (log.isDebugEnabled()) {
            log.debug("Waiting for first searcher of core {}, id: {} to be registered", core.getName(), core);
          }
          try {
            waitSearcher[0].get();
          } catch (ExecutionException e) {
            log.warn("Wait for a searcher to be registered for core {}, id: {} failed due to: {}", core.getName(), core, e, e);
            success = false;
          }
        }
        if (success)  {
          if (searcher == null) {
            // should never happen
            if (log.isDebugEnabled()) {
              log.debug("Did not find a searcher even after the future callback for core: {}, id: {}!!!", core.getName(), core);
            }
          } else  {
            if (log.isInfoEnabled()) {
              log.info("Found a registered searcher: {}, took: {} ms for core: {}, id: {}", searcher.get(), timer.getTime(), core.getName(), core);
            }
          }
        }
      } finally {
        if (searcher != null) {
          searcher.decref();
        }
      }
    }
    RefCounted<SolrIndexSearcher> newestSearcher = core.getNewestSearcher(false);
    if (newestSearcher != null) {
      if (log.isDebugEnabled()) {
        log.debug("Found newest searcher: {} for core: {}, id: {}", newestSearcher.get(), core.getName(), core);
      }
      newestSearcher.decref();
    }
  }
}
 
Example 23
Source Project: lucene-solr   Source File: TestIndexingPerformance.java    License: Apache License 2.0 4 votes vote down vote up
public void testIndexingPerf() throws IOException {
  int iter=1000;
  String iterS = System.getProperty("iter");
  if (iterS != null) iter=Integer.parseInt(iterS);
  boolean overwrite = Boolean.parseBoolean(System.getProperty("overwrite","false"));
  String doc = System.getProperty("doc");
  if (doc != null) {
    StrUtils.splitSmart(doc,",",true);
  }


  SolrQueryRequest req = lrf.makeRequest();
  UpdateHandler updateHandler = req.getCore().getUpdateHandler();
  String field = "textgap";

  String[] fields = {field,"simple"
          ,field,"test"
          ,field,"how now brown cow"
          ,field,"what's that?"
          ,field,"radical!"
          ,field,"what's all this about, anyway?"
          ,field,"just how fast is this text indexing?"
  };


/***
  String[] fields = {
          "a_i","1"
          ,"b_i","2"
          ,"c_i","3"
          ,"d_i","4"
          ,"e_i","5"
          ,"f_i","6"
          ,"g_i","7"
          ,"h_i","8"
          ,"i_i","9"
          ,"j_i","0"
          ,"k_i","0"
  };
 ***/

  final RTimer timer = new RTimer();

  AddUpdateCommand add = new AddUpdateCommand(req);
  add.overwrite = overwrite;

  for (int i=0; i<iter; i++) {
    add.clear();
    add.solrDoc = new SolrInputDocument();
    add.solrDoc.addField("id", Integer.toString(i));
    for (int j=0; j<fields.length; j+=2) {
      String f = fields[j];
      String val = fields[j+1];
      add.solrDoc.addField(f, val);
    }
    updateHandler.addDoc(add);
  }
  if (log.isInfoEnabled()) {
    log.info("doc={}", Arrays.toString(fields));
  }
  double elapsed = timer.getTime();
  if (log.isInfoEnabled()) {
    log.info("iter={} time={} throughput={}", iter, elapsed, ((long) iter * 1000) / elapsed);
  }

  //discard all the changes
  updateHandler.rollback(new RollbackUpdateCommand(req));

  req.close();
}
 
Example 24
Source Project: lucene-solr   Source File: TestSolrJ.java    License: Apache License 2.0 4 votes vote down vote up
public static void main(String[] args) throws Exception {
  // String addr = "http://odin.local:80/solr";
  // String addr = "http://odin.local:8983/solr";
  String addr = "http://127.0.0.1:8983/solr";

  int i = 0;
  final int nDocs = Integer.parseInt(args[i++]);
  final int nProducers = Integer.parseInt(args[i++]);
  final int nConnections = Integer.parseInt(args[i++]);
  final int maxSleep = Integer.parseInt(args[i++]);

  ConcurrentUpdateSolrClient concurrentClient = null;

  // server = concurrentClient = new ConcurrentUpdateSolrServer(addr,32,8);
  client = concurrentClient = getConcurrentUpdateSolrClient(addr,64,nConnections);

  client.deleteByQuery("*:*");
  client.commit();

  final RTimer timer = new RTimer();

  final int docsPerThread = nDocs / nProducers;

  Thread[] threads = new Thread[nProducers];

  for (int threadNum = 0; threadNum<nProducers; threadNum++) {
    final int base = threadNum * docsPerThread;

    threads[threadNum] = new Thread("add-thread"+i) {
      @Override
      public void run(){
        try {
          indexDocs(base, docsPerThread, maxSleep);
        } catch (Exception e) {
          System.out.println("###############################CAUGHT EXCEPTION");
          e.printStackTrace();
          ex = e;
        }
      }
    };
    threads[threadNum].start();
  }

  // optional: wait for commit?

  for (int threadNum = 0; threadNum<nProducers; threadNum++) {
    threads[threadNum].join();
  }

  if (concurrentClient != null) {
    concurrentClient.blockUntilFinished();
  }

  double elapsed = timer.getTime();
  System.out.println("time="+elapsed + " throughput="+(nDocs*1000/elapsed) + " Exception="+ex);

  // should server threads be marked as daemon?
  // need a server.close()!!!
}
 
Example 25
Source Project: lucene-solr   Source File: HttpPartitionTest.java    License: Apache License 2.0 4 votes vote down vote up
protected void waitToSeeReplicasActive(String testCollectionName, String shardId, Set<String> replicasToCheck, int maxWaitSecs) throws Exception {
  final RTimer timer = new RTimer();

  ZkStateReader zkr = cloudClient.getZkStateReader();
  zkr.forceUpdateCollection(testCollectionName);
  ClusterState cs = zkr.getClusterState();
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    cs = cloudClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    final DocCollection docCollection = cs.getCollectionOrNull(testCollectionName);
    assertNotNull(docCollection);
    Slice shard = docCollection.getSlice(shardId);
    assertNotNull("No Slice for "+shardId, shard);
    allReplicasUp = true; // assume true

    // wait to see all replicas are "active"
    for (Replica replica : shard.getReplicas()) {
      if (!replicasToCheck.contains(replica.getName()))
        continue;

      final Replica.State state = replica.getState();
      if (state != Replica.State.ACTIVE) {
        if (log.isInfoEnabled()) {
          log.info("Replica {} is currently {}", replica.getName(), state);
        }
        allReplicasUp = false;
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(200L);
      } catch (Exception ignoreMe) {}
      waitMs += 200L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see replicas "+ replicasToCheck +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  if (log.isInfoEnabled()) {
    log.info("Took {} ms to see replicas [{}] become active.", timer.getTime(), replicasToCheck);
  }
}
 
Example 26
Source Project: SolRDF   Source File: Sparql11SearchHandler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RTimer getRequestTimer() {
	return request.getRequestTimer();
}
 
Example 27
Source Project: SolRDF   Source File: Sparql11SearchHandler.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public RTimer getRequestTimer() {
	return request.getRequestTimer();
}