org.apache.lucene.index.MergeScheduler Java Examples

The following examples show how to use org.apache.lucene.index.MergeScheduler. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** just tries to configure things to keep the open file
 * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
  // keep number of open files lowish
  MergePolicy mp = w.getConfig().getMergePolicy();
  mp.setNoCFSRatio(1.0);
  if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
  } else if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
    tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
  }
  MergeScheduler ms = w.getConfig().getMergeScheduler();
  if (ms instanceof ConcurrentMergeScheduler) {
    // wtf... shouldnt it be even lower since it's 1 by default?!?!
    ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
  }
}
 
Example #2
Source File: SharedMergeScheduler.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
public MergeScheduler getMergeScheduler() {
  return new MergeScheduler() {

    private final String _id = UUID.randomUUID().toString();

    @Override
    public void merge(IndexWriter writer) throws IOException {
      addMerges(_id, writer);
    }

    @Override
    public void close() throws IOException {
      remove(_id);
    }

    @Override
    public MergeScheduler clone() {
      return getMergeScheduler();
    }
  };
}
 
Example #3
Source File: SolrIndexConfig.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException {
  IndexSchema schema = core.getLatestSchema();
  IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core));
  if (maxBufferedDocs != -1)
    iwc.setMaxBufferedDocs(maxBufferedDocs);

  if (ramBufferSizeMB != -1)
    iwc.setRAMBufferSizeMB(ramBufferSizeMB);

  if (ramPerThreadHardLimitMB != -1) {
    iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);
  }

  iwc.setSimilarity(schema.getSimilarity());
  MergePolicy mergePolicy = buildMergePolicy(core.getResourceLoader(), schema);
  iwc.setMergePolicy(mergePolicy);
  MergeScheduler mergeScheduler = buildMergeScheduler(core.getResourceLoader());
  iwc.setMergeScheduler(mergeScheduler);
  iwc.setInfoStream(infoStream);

  if (mergePolicy instanceof SortingMergePolicy) {
    Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort();
    iwc.setIndexSort(indexSort);
  }

  iwc.setUseCompoundFile(useCompoundFile);

  if (mergedSegmentWarmerInfo != null) {
    // TODO: add infostream -> normal logging system (there is an issue somewhere)
    @SuppressWarnings({"rawtypes"})
    IndexReaderWarmer warmer = core.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className,
                                                                      IndexReaderWarmer.class,
                                                                      null,
                                                                      new Class[] { InfoStream.class },
                                                                      new Object[] { iwc.getInfoStream() });
    iwc.setMergedSegmentWarmer(warmer);
  }

  return iwc;
}
 
Example #4
Source File: SolrIndexConfig.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
private MergeScheduler buildMergeScheduler(SolrResourceLoader resourceLoader) {
  String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
  MergeScheduler scheduler = resourceLoader.newInstance(msClassName, MergeScheduler.class);

  if (mergeSchedulerInfo != null) {
    // LUCENE-5080: these two setters are removed, so we have to invoke setMaxMergesAndThreads
    // if someone has them configured.
    if (scheduler instanceof ConcurrentMergeScheduler) {
      @SuppressWarnings({"rawtypes"})
      NamedList args = mergeSchedulerInfo.initArgs.clone();
      Integer maxMergeCount = (Integer) args.remove("maxMergeCount");
      if (maxMergeCount == null) {
        maxMergeCount = ((ConcurrentMergeScheduler) scheduler).getMaxMergeCount();
      }
      Integer maxThreadCount = (Integer) args.remove("maxThreadCount");
      if (maxThreadCount == null) {
        maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount();
      }
      ((ConcurrentMergeScheduler)scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
      Boolean ioThrottle = (Boolean) args.remove("ioThrottle");
      if (ioThrottle != null && !ioThrottle) { //by-default 'enabled'
          ((ConcurrentMergeScheduler) scheduler).disableAutoIOThrottle();
      }
      SolrPluginUtils.invokeSetters(scheduler, args);
    } else {
      SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
    }
  }

  return scheduler;
}
 
Example #5
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public static void syncConcurrentMerges(MergeScheduler ms) {
  if (ms instanceof ConcurrentMergeScheduler)
    ((ConcurrentMergeScheduler) ms).sync();
}
 
Example #6
Source File: ElasticsearchConcurrentMergeScheduler.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public MergeScheduler clone() {
    // Lucene IW makes a clone internally but since we hold on to this instance
    // the clone will just be the identity.
    return this;
}