org.apache.lucene.index.MergePolicy Java Examples

The following examples show how to use org.apache.lucene.index.MergePolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SolrIndexConfigTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSortingMPSolrIndexConfigCreation() throws Exception {
  final String expectedFieldName = "timestamp_i_dvo";
  final SortField.Type expectedFieldType = SortField.Type.INT;
  final boolean expectedFieldSortDescending = true;

  SolrConfig solrConfig = new SolrConfig(instanceDir, solrConfigFileNameSortingMergePolicyFactory);
  SolrIndexConfig solrIndexConfig = new SolrIndexConfig(solrConfig, null, null);
  assertNotNull(solrIndexConfig);
  IndexSchema indexSchema = IndexSchemaFactory.buildIndexSchema(schemaFileName, solrConfig);

  h.getCore().setLatestSchema(indexSchema);
  IndexWriterConfig iwc = solrIndexConfig.toIndexWriterConfig(h.getCore());

  final MergePolicy mergePolicy = iwc.getMergePolicy();
  assertNotNull("null mergePolicy", mergePolicy);
  assertTrue("mergePolicy ("+mergePolicy+") is not a SortingMergePolicy", mergePolicy instanceof SortingMergePolicy);
  final SortingMergePolicy sortingMergePolicy = (SortingMergePolicy) mergePolicy;
  final Sort expected = new Sort(new SortField(expectedFieldName, expectedFieldType, expectedFieldSortDescending));
  final Sort actual = sortingMergePolicy.getSort();
  assertEquals("SortingMergePolicy.getSort", expected, actual);
}
 
Example #2
Source File: InternalEngine.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
protected void handleMergeException(final Directory dir, final Throwable exc) {
    logger.error("failed to merge", exc);
    if (config().getMergeSchedulerConfig().isNotifyOnMergeFailure()) {
        engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
            @Override
            public void onFailure(Throwable t) {
                logger.debug("merge failure action rejected", t);
            }

            @Override
            protected void doRun() throws Exception {
                MergePolicy.MergeException e = new MergePolicy.MergeException(exc, dir);
                failEngine("merge failed", e);
            }
        });
    }
}
 
Example #3
Source File: DLBasedEngine.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
protected void handleMergeException(final Directory dir, final Throwable exc) {
    logger.error("failed to merge", exc);
    if (config().getMergeSchedulerConfig().isNotifyOnMergeFailure()) {
        engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
            @Override
            public void onFailure(Throwable t) {
                logger.debug("merge failure action rejected", t);
            }

            @Override
            protected void doRun() throws Exception {
                MergePolicy.MergeException e = new MergePolicy.MergeException(exc, dir);
                failEngine("merge failed", e);
            }
        });
    }
}
 
Example #4
Source File: SortingMergePolicyFactory.java    From linden with Apache License 2.0 6 votes vote down vote up
@Override
public MergePolicy getInstance(Map<String, String> params) throws IOException {
  String field = params.get(SORT_FIELD);
  SortField.Type sortFieldType = SortField.Type.DOC;
  if (params.containsKey(SORT_FIELD_TYPE)) {
    sortFieldType = SortField.Type.valueOf(params.get(SORT_FIELD_TYPE).toUpperCase());
  }

  if (sortFieldType == SortField.Type.DOC) {
    throw new IOException(
        "Relying on internal lucene DocIDs is not guaranteed to work, this is only an implementation detail.");
  }

  boolean desc = true;
  if (params.containsKey(SORT_DESC)) {
    try {
      desc = Boolean.valueOf(params.get(SORT_DESC));
    } catch (Exception e) {
      desc = true;
    }
  }
  SortField sortField = new SortField(field, sortFieldType, desc);
  Sort sort = new Sort(sortField);
  return new SortingMergePolicyDecorator(new TieredMergePolicy(), sort);
}
 
Example #5
Source File: EngineTestCase.java    From crate with Apache License 2.0 6 votes vote down vote up
protected InternalEngine createEngine(
    IndexSettings indexSettings,
    Store store,
    Path translogPath,
    MergePolicy mergePolicy,
    @Nullable IndexWriterFactory indexWriterFactory,
    @Nullable BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier,
    @Nullable LongSupplier globalCheckpointSupplier,
    @Nullable ToLongBiFunction<Engine, Engine.Operation> seqNoForOperation) throws IOException {
    return createEngine(
        indexSettings,
        store,
        translogPath,
        mergePolicy,
        indexWriterFactory,
        localCheckpointTrackerSupplier,
        seqNoForOperation,
        globalCheckpointSupplier);
}
 
Example #6
Source File: SimplePrimaryNode.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static IndexWriter initWriter(int id, Random random, Path indexPath, boolean doCheckIndexOnClose) throws IOException {
  Directory dir = SimpleReplicaNode.getDirectory(random, id, indexPath, doCheckIndexOnClose);

  MockAnalyzer analyzer = new MockAnalyzer(random);
  analyzer.setMaxTokenLength(TestUtil.nextInt(random, 1, IndexWriter.MAX_TERM_LENGTH));
  IndexWriterConfig iwc = LuceneTestCase.newIndexWriterConfig(random, analyzer);

  MergePolicy mp = iwc.getMergePolicy();
  //iwc.setInfoStream(new PrintStreamInfoStream(System.out));

  // Force more frequent merging so we stress merge warming:
  if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setSegmentsPerTier(3);
    tmp.setMaxMergeAtOnce(3);
  } else if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(3);
  }

  IndexWriter writer = new IndexWriter(dir, iwc);

  TestUtil.reduceOpenFiles(writer);
  return writer;
}
 
Example #7
Source File: SolrIndexConfig.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Builds a MergePolicy using the configured MergePolicyFactory
 * or if no factory is configured uses the configured mergePolicy PluginInfo.
 */
@SuppressWarnings({"unchecked", "rawtypes"})
private MergePolicy buildMergePolicy(SolrResourceLoader resourceLoader, IndexSchema schema) {

  final String mpfClassName;
  final MergePolicyFactoryArgs mpfArgs;
  if (mergePolicyFactoryInfo == null) {
    mpfClassName = DEFAULT_MERGE_POLICY_FACTORY_CLASSNAME;
    mpfArgs = new MergePolicyFactoryArgs();
  } else {
    mpfClassName = mergePolicyFactoryInfo.className;
    mpfArgs = new MergePolicyFactoryArgs(mergePolicyFactoryInfo.initArgs);
  }

  final MergePolicyFactory mpf = resourceLoader.newInstance(
      mpfClassName,
      MergePolicyFactory.class,
      NO_SUB_PACKAGES,
      new Class[] { SolrResourceLoader.class, MergePolicyFactoryArgs.class, IndexSchema.class },
      new Object[] {resourceLoader, mpfArgs, schema });

  return mpf.getMergePolicy();
}
 
Example #8
Source File: SegmentsInfoRequestHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException {
  SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
  RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
  try {
    IndexWriter indexWriter = refCounted.get();
    if (indexWriter instanceof SolrIndexWriter) {
      result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges());
    }
    //get chosen merge policy
    MergePolicy mp = indexWriter.getConfig().getMergePolicy();
    //Find merges
    MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
    if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
      for (OneMerge merge : findMerges.merges) {
        //TODO: add merge grouping
        for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
          mergeCandidates.add(mergeSegmentInfo.info.name);
        }
      }
    }

    return result;
  } finally {
    refCounted.decref();
  }
}
 
Example #9
Source File: InternalEngine.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
protected void handleMergeException(final Directory dir, final Throwable exc) {
    engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
        @Override
        public void onFailure(Exception e) {
            logger.debug("merge failure action rejected", e);
        }

        @Override
        protected void doRun() throws Exception {
            /*
             * We do this on another thread rather than the merge thread that we are initially called on so that we have complete
             * confidence that the call stack does not contain catch statements that would cause the error that might be thrown
             * here from being caught and never reaching the uncaught exception handler.
             */
            failEngine("merge failed", new MergePolicy.MergeException(exc, dir));
        }
    });
}
 
Example #10
Source File: WrapperMergePolicyFactoryTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testProperlyInitializesWrappedMergePolicy() {
  final TieredMergePolicy defaultTMP = new TieredMergePolicy();
  final int testMaxMergeAtOnce = defaultTMP.getMaxMergeAtOnce() * 2;
  final double testMaxMergedSegmentMB = defaultTMP.getMaxMergedSegmentMB() * 10;

  final MergePolicyFactoryArgs args = new MergePolicyFactoryArgs();
  args.add(WrapperMergePolicyFactory.WRAPPED_PREFIX, "test");
  args.add("test.class", TieredMergePolicyFactory.class.getName());
  args.add("test.maxMergeAtOnce", testMaxMergeAtOnce);
  args.add("test.maxMergedSegmentMB", testMaxMergedSegmentMB);
  MergePolicyFactory mpf = new DefaultingWrapperMergePolicyFactory(resourceLoader, args, null) {
    @Override
    protected MergePolicy getDefaultWrappedMergePolicy() {
      throw new IllegalStateException("Should not have reached here!");
    }
  };
  final MergePolicy mp = mpf.getMergePolicy();
  assertSame(mp.getClass(), TieredMergePolicy.class);
  final TieredMergePolicy tmp = (TieredMergePolicy)mp;
  assertEquals("maxMergeAtOnce", testMaxMergeAtOnce, tmp.getMaxMergeAtOnce());
  assertEquals("maxMergedSegmentMB", testMaxMergedSegmentMB, tmp.getMaxMergedSegmentMB(), 0.0d);
}
 
Example #11
Source File: test.java    From vscode-extension with MIT License 6 votes vote down vote up
@Override
protected void handleMergeException(final Directory dir, final Throwable exc) {
    engineConfig.getThreadPool().generic().execute(new AbstractRunnable() {
        @Override
        public void onFailure(Exception e) {
            logger.debug("merge failure action rejected", e);
        }

        @Override
        protected void doRun() throws Exception {
            /*
             * We do this on another thread rather than the merge thread that we are initially called on so that we have complete
             * confidence that the call stack does not contain catch statements that would cause the error that might be thrown
             * here from being caught and never reaching the uncaught exception handler.
             */
            failEngine("merge failed", new MergePolicy.MergeException(exc, dir));
        }
    });
}
 
Example #12
Source File: TestUtil.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** just tries to configure things to keep the open file
 * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
  // keep number of open files lowish
  MergePolicy mp = w.getConfig().getMergePolicy();
  mp.setNoCFSRatio(1.0);
  if (mp instanceof LogMergePolicy) {
    LogMergePolicy lmp = (LogMergePolicy) mp;
    lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
  } else if (mp instanceof TieredMergePolicy) {
    TieredMergePolicy tmp = (TieredMergePolicy) mp;
    tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
    tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
  }
  MergeScheduler ms = w.getConfig().getMergeScheduler();
  if (ms instanceof ConcurrentMergeScheduler) {
    // wtf... shouldnt it be even lower since it's 1 by default?!?!
    ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
  }
}
 
Example #13
Source File: SolrIndexConfig.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException {
  IndexSchema schema = core.getLatestSchema();
  IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core));
  if (maxBufferedDocs != -1)
    iwc.setMaxBufferedDocs(maxBufferedDocs);

  if (ramBufferSizeMB != -1)
    iwc.setRAMBufferSizeMB(ramBufferSizeMB);

  if (ramPerThreadHardLimitMB != -1) {
    iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB);
  }

  iwc.setSimilarity(schema.getSimilarity());
  MergePolicy mergePolicy = buildMergePolicy(core.getResourceLoader(), schema);
  iwc.setMergePolicy(mergePolicy);
  MergeScheduler mergeScheduler = buildMergeScheduler(core.getResourceLoader());
  iwc.setMergeScheduler(mergeScheduler);
  iwc.setInfoStream(infoStream);

  if (mergePolicy instanceof SortingMergePolicy) {
    Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort();
    iwc.setIndexSort(indexSort);
  }

  iwc.setUseCompoundFile(useCompoundFile);

  if (mergedSegmentWarmerInfo != null) {
    // TODO: add infostream -> normal logging system (there is an issue somewhere)
    @SuppressWarnings({"rawtypes"})
    IndexReaderWarmer warmer = core.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className,
                                                                      IndexReaderWarmer.class,
                                                                      null,
                                                                      new Class[] { InfoStream.class },
                                                                      new Object[] { iwc.getInfoStream() });
    iwc.setMergedSegmentWarmer(warmer);
  }

  return iwc;
}
 
Example #14
Source File: WrapperMergePolicyFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Returns a wrapping {@link MergePolicy} with its set parameters configured. */
@Override
public final MergePolicy getMergePolicy() {
  final MergePolicy wrappedMP = getWrappedMergePolicy();
  final MergePolicy mp = getMergePolicyInstance(wrappedMP);
  args.invokeSetters(mp);
  return mp;
}
 
Example #15
Source File: WrapperMergePolicyFactoryTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void implTestUpgradeIndexMergePolicyFactory(Double wrappingNoCFSRatio, Double wrappedNoCFSRatio) {
  final MergePolicyFactoryArgs args = new MergePolicyFactoryArgs();
  if (wrappingNoCFSRatio != null) {
    args.add("noCFSRatio", wrappingNoCFSRatio); // noCFSRatio for the wrapping merge policy
  }
  args.add(WrapperMergePolicyFactory.WRAPPED_PREFIX, "wrapped");
  args.add("wrapped.class", TieredMergePolicyFactory.class.getName());
  if (wrappedNoCFSRatio != null) {
    args.add("wrapped.noCFSRatio", wrappedNoCFSRatio); // noCFSRatio for the wrapped merge policy
  }

  MergePolicyFactory mpf;
  try {
    mpf = new UpgradeIndexMergePolicyFactory(resourceLoader, args, null);
    assertFalse("Should only reach here if wrapping and wrapped args don't overlap!",
        (wrappingNoCFSRatio != null && wrappedNoCFSRatio != null));

    for (int ii=1; ii<=2; ++ii) { // it should be okay to call getMergePolicy() more than once
      final MergePolicy mp = mpf.getMergePolicy();
      if (wrappingNoCFSRatio != null) {
        assertEquals("#"+ii+" wrappingNoCFSRatio", wrappingNoCFSRatio.doubleValue(), mp.getNoCFSRatio(), 0.0d);
      }
      if (wrappedNoCFSRatio != null) {
        assertEquals("#"+ii+" wrappedNoCFSRatio", wrappedNoCFSRatio.doubleValue(), mp.getNoCFSRatio(), 0.0d);
      }
      assertSame(mp.getClass(), UpgradeIndexMergePolicy.class);
    }

  } catch (IllegalArgumentException iae) {
    assertEquals("Wrapping and wrapped merge policy args overlap! [noCFSRatio]", iae.getMessage());
    assertTrue("Should only reach here if wrapping and wrapped args do overlap!",
        (wrappingNoCFSRatio != null && wrappedNoCFSRatio != null));
  }
}
 
Example #16
Source File: TestMergeSchedulerExternal.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected MergeThread getMergeThread(MergeSource mergeSource, MergePolicy.OneMerge merge) throws IOException {
  MergeThread thread = new MyMergeThread(mergeSource, merge);
  thread.setDaemon(true);
  thread.setName("MyMergeThread");
  return thread;
}
 
Example #17
Source File: EngineTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
public EngineConfig copy(EngineConfig config, MergePolicy mergePolicy) {
    return new EngineConfig(config.getShardId(), config.getAllocationId(), config.getThreadPool(), config.getIndexSettings(),
        config.getStore(), mergePolicy, config.getAnalyzer(),
        new CodecService(null, logger), config.getEventListener(), config.getQueryCache(), config.getQueryCachingPolicy(),
        config.getTranslogConfig(), config.getFlushMergesAfter(),
        config.getExternalRefreshListener(), Collections.emptyList(),
        config.getCircuitBreakerService(), config.getGlobalCheckpointSupplier(), config.getPrimaryTermSupplier(),
        config.getTombstoneDocSupplier());
}
 
Example #18
Source File: IndexMergeTool.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static Options parse(String args[]) throws ReflectiveOperationException {
  Options options = new Options();
  int index = 0;
  while (index < args.length) {
    if (!args[index].startsWith("-")) {
      break;
    }
    if (args[index] == "--") {
      break;
    }
    switch(args[index]) {
      case "-merge-policy":
        String clazzName = args[++index];
        Class<? extends MergePolicy> clazz = Class.forName(clazzName).asSubclass(MergePolicy.class);
        options.config.setMergePolicy(clazz.getConstructor().newInstance());
        break;
      case "-max-segments":
        options.maxSegments = Integer.parseInt(args[++index]);
        break;
      case "-verbose":
        options.config.setInfoStream(System.err);
        break;
      default: throw new IllegalArgumentException("unrecognized option: '" + args[index] + "'\n" + USAGE);
    }
    index++;
  }

  // process any remaining arguments as the target and source index paths.
  int numPaths = args.length - index;
  if (numPaths < 3) {
    throw new IllegalArgumentException("not enough parameters.\n" + USAGE);
  }

  options.mergedIndexPath = args[index];
  options.indexPaths = new String[numPaths - 1];
  System.arraycopy(args, index + 1, options.indexPaths, 0, options.indexPaths.length);
  return options;
}
 
Example #19
Source File: TieredMergePolicyFactory.java    From linden with Apache License 2.0 5 votes vote down vote up
@Override
public MergePolicy getInstance(Map<String, String> config) throws IOException {
  TieredMergePolicy mergePolicy = new TieredMergePolicy();

  if (config.containsKey(SEGMENTS_PER_TIER)) {
    mergePolicy.setSegmentsPerTier(Double.valueOf(config.get(SEGMENTS_PER_TIER)));
  }
  if (config.containsKey(MAX_MERGE_AT_ONCE)) {
    mergePolicy.setMaxMergeAtOnce(Integer.valueOf(config.get(MAX_MERGE_AT_ONCE)));
  }
  return mergePolicy;
}
 
Example #20
Source File: LindenConfig.java    From linden with Apache License 2.0 5 votes vote down vote up
public IndexWriterConfig createIndexWriterConfig() throws IOException {
  IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LATEST, getIndexAnalyzerInstance());
  indexWriterConfig.setRAMBufferSizeMB(48);

  MergePolicy mergePolicy = getPluginManager().getInstance(LindenConfigBuilder.MERGE_POLICY, MergePolicy.class);
  if (mergePolicy != null) {
    indexWriterConfig.setMergePolicy(mergePolicy);
  }
  LOGGER.info("Merge policy : {}", mergePolicy == null ? "Default" : mergePolicy);

  ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
  cms.setMaxMergesAndThreads(8, 1);
  indexWriterConfig.setMergeScheduler(cms);
  return indexWriterConfig;
}
 
Example #21
Source File: EngineTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
protected InternalEngine createEngine(
    IndexSettings indexSettings,
    Store store,
    Path translogPath,
    MergePolicy mergePolicy,
    @Nullable IndexWriterFactory indexWriterFactory,
    @Nullable BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier,
    @Nullable ToLongBiFunction<Engine, Engine.Operation> seqNoForOperation,
    @Nullable LongSupplier globalCheckpointSupplier) throws IOException {
    EngineConfig config = config(indexSettings, store, translogPath, mergePolicy, null, globalCheckpointSupplier);
    return createEngine(indexWriterFactory, localCheckpointTrackerSupplier, seqNoForOperation, config);
}
 
Example #22
Source File: EngineTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
protected InternalEngine createEngine(
    IndexSettings indexSettings,
    Store store,
    Path translogPath,
    MergePolicy mergePolicy,
    @Nullable IndexWriterFactory indexWriterFactory,
    @Nullable BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier,
    @Nullable LongSupplier globalCheckpointSupplier) throws IOException {
    return createEngine(
        indexSettings, store, translogPath, mergePolicy, indexWriterFactory, localCheckpointTrackerSupplier, null,
        globalCheckpointSupplier);
}
 
Example #23
Source File: DefaultSolrCoreState.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public Sort getMergePolicySort() throws IOException {
  lock(iwLock.readLock());
  try {
    if (indexWriter != null) {
      final MergePolicy mergePolicy = indexWriter.getConfig().getMergePolicy();
      if (mergePolicy instanceof SortingMergePolicy) {
        return ((SortingMergePolicy)mergePolicy).getSort();
      }
    }
  } finally {
    iwLock.readLock().unlock();
  }
  return null;
}
 
Example #24
Source File: test.java    From vscode-extension with MIT License 5 votes vote down vote up
private IndexWriterConfig getIndexWriterConfig() {
    final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
    iwc.setCommitOnClose(false); // we by default don't commit on close
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
    // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
    boolean verbose = false;
    try {
        verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
    } catch (Exception ignore) {
    }
    iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
    iwc.setMergeScheduler(mergeScheduler);
    // Give us the opportunity to upgrade old segments while performing
    // background merges
    MergePolicy mergePolicy = config().getMergePolicy();
    // always configure soft-deletes field so an engine with soft-deletes disabled can open a Lucene index with soft-deletes.
    iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD);
    if (softDeleteEnabled) {
        mergePolicy = new RecoverySourcePruneMergePolicy(SourceFieldMapper.RECOVERY_SOURCE_NAME, softDeletesPolicy::getRetentionQuery,
            new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery, mergePolicy));
    }
    iwc.setMergePolicy(new ElasticsearchMergePolicy(mergePolicy));
    iwc.setSimilarity(engineConfig.getSimilarity());
    iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
    iwc.setCodec(engineConfig.getCodec());
    iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
    if (config().getIndexSort() != null) {
        iwc.setIndexSort(config().getIndexSort());
    }
    return iwc;
}
 
Example #25
Source File: EngineConfig.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@link org.elasticsearch.index.engine.EngineConfig}
 */
public EngineConfig(ShardId shardId, ThreadPool threadPool, ShardIndexingService indexingService,
                    Settings indexSettings, IndicesWarmer warmer, Store store, SnapshotDeletionPolicy deletionPolicy,
                    MergePolicy mergePolicy, MergeSchedulerConfig mergeSchedulerConfig, Analyzer analyzer,
                    Similarity similarity, CodecService codecService, Engine.FailedEngineListener failedEngineListener,
                    TranslogRecoveryPerformer translogRecoveryPerformer, QueryCache queryCache, QueryCachingPolicy queryCachingPolicy, IndexSearcherWrappingService wrappingService, TranslogConfig translogConfig) {
    this.shardId = shardId;
    this.indexSettings = indexSettings;
    this.threadPool = threadPool;
    this.indexingService = indexingService;
    this.warmer = warmer;
    this.store = store;
    this.deletionPolicy = deletionPolicy;
    this.mergePolicy = mergePolicy;
    this.mergeSchedulerConfig = mergeSchedulerConfig;
    this.analyzer = analyzer;
    this.similarity = similarity;
    this.codecService = codecService;
    this.failedEngineListener = failedEngineListener;
    this.wrappingService = wrappingService;
    this.optimizeAutoGenerateId = indexSettings.getAsBoolean(EngineConfig.INDEX_OPTIMIZE_AUTOGENERATED_ID_SETTING, false);
    this.compoundOnFlush = indexSettings.getAsBoolean(EngineConfig.INDEX_COMPOUND_ON_FLUSH, compoundOnFlush);
    codecName = indexSettings.get(EngineConfig.INDEX_CODEC_SETTING, EngineConfig.DEFAULT_CODEC_NAME);
    // We start up inactive and rely on IndexingMemoryController to give us our fair share once we start indexing:
    indexingBufferSize = IndexingMemoryController.INACTIVE_SHARD_INDEXING_BUFFER;
    gcDeletesInMillis = indexSettings.getAsTime(INDEX_GC_DELETES_SETTING, EngineConfig.DEFAULT_GC_DELETES).millis();
    versionMapSizeSetting = indexSettings.get(INDEX_VERSION_MAP_SIZE, DEFAULT_VERSION_MAP_SIZE);
    updateVersionMapSize();
    this.translogRecoveryPerformer = translogRecoveryPerformer;
    this.forceNewTranslog = indexSettings.getAsBoolean(INDEX_FORCE_NEW_TRANSLOG, false);
    this.queryCache = queryCache;
    this.queryCachingPolicy = queryCachingPolicy;
    this.translogConfig = translogConfig;
}
 
Example #26
Source File: InternalEngine.java    From crate with Apache License 2.0 5 votes vote down vote up
private IndexWriterConfig getIndexWriterConfig() {
    final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
    iwc.setCommitOnClose(false); // we by default don't commit on close
    iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND);
    iwc.setReaderAttributes(getReaderAttributes(store.directory()));
    iwc.setIndexDeletionPolicy(combinedDeletionPolicy);
    // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
    boolean verbose = false;
    try {
        verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
    } catch (Exception ignore) {
        // ignored
    }
    iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
    iwc.setMergeScheduler(mergeScheduler);
    // Give us the opportunity to upgrade old segments while performing
    // background merges
    MergePolicy mergePolicy = config().getMergePolicy();
    // always configure soft-deletes field so an engine with soft-deletes disabled can open a Lucene index with soft-deletes.
    iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD);
    if (softDeleteEnabled) {
        mergePolicy = new RecoverySourcePruneMergePolicy(SourceFieldMapper.RECOVERY_SOURCE_NAME, softDeletesPolicy::getRetentionQuery,
            new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery, mergePolicy));
    }
    iwc.setMergePolicy(new ElasticsearchMergePolicy(mergePolicy));
    iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
    iwc.setCodec(engineConfig.getCodec());
    iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
    return iwc;
}
 
Example #27
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 5 votes vote down vote up
RecoverySourcePruneMergePolicy(String recoverySourceField, Supplier<Query> retainSourceQuerySupplier, MergePolicy in) {
    super(in, toWrap -> new OneMerge(toWrap.segments) {
        @Override
        public CodecReader wrapForMerge(CodecReader reader) throws IOException {
            CodecReader wrapped = toWrap.wrapForMerge(reader);
            return wrapReader(recoverySourceField, wrapped, retainSourceQuerySupplier);
        }
    });
}
 
Example #28
Source File: ElasticsearchConcurrentMergeScheduler.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings("sync-override")
protected MergeThread getMergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
    MergeThread thread = super.getMergeThread(writer, merge);
    thread.setName(EsExecutors.threadName(indexSettings, "[" + shardId.getIndexName() + "][" + shardId.id() + "]: " + thread.getName()));
    return thread;
}
 
Example #29
Source File: WrapperMergePolicyFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Returns an instance of the wrapped {@link MergePolicy} after it has been configured with all set parameters. */
@SuppressWarnings({"rawtypes"})
protected final MergePolicy getWrappedMergePolicy() {
  if (wrappedMergePolicyArgs == null) {
    return getDefaultWrappedMergePolicy();
  }

  final MergePolicyFactory mpf = resourceLoader.newInstance(
      wrappedMergePolicyClassName,
      MergePolicyFactory.class,
      NO_SUB_PACKAGES,
      new Class[] {SolrResourceLoader.class, MergePolicyFactoryArgs.class, IndexSchema.class},
      new Object[] {resourceLoader, wrappedMergePolicyArgs, schema});
  return mpf.getMergePolicy();
}
 
Example #30
Source File: IndexSettings.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the merge policy that should be used for this index.
 */
public MergePolicy getMergePolicy() {
    return mergePolicyConfig.getMergePolicy();
}