org.apache.lucene.index.IndexDeletionPolicy Java Examples

The following examples show how to use org.apache.lucene.index.IndexDeletionPolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexAndTaxonomyRevision.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Constructor over the given {@link IndexWriter}. Uses the last
 * {@link IndexCommit} found in the {@link Directory} managed by the given
 * writer.
 */
public IndexAndTaxonomyRevision(IndexWriter indexWriter, SnapshotDirectoryTaxonomyWriter taxoWriter)
    throws IOException {
  IndexDeletionPolicy delPolicy = indexWriter.getConfig().getIndexDeletionPolicy();
  if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
    throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
  }
  this.indexWriter = indexWriter;
  this.taxoWriter = taxoWriter;
  this.indexSDP = (SnapshotDeletionPolicy) delPolicy;
  this.taxoSDP = taxoWriter.getDeletionPolicy();
  this.indexCommit = indexSDP.snapshot();
  this.taxoCommit = taxoSDP.snapshot();
  this.version = revisionVersion(indexCommit, taxoCommit);
  this.sourceFiles = revisionFiles(indexCommit, taxoCommit);
}
 
Example #2
Source File: SolrCore.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private IndexDeletionPolicyWrapper initDeletionPolicy(IndexDeletionPolicyWrapper delPolicyWrapper) {
  if (delPolicyWrapper != null) {
    return delPolicyWrapper;
  }

  final PluginInfo info = solrConfig.getPluginInfo(IndexDeletionPolicy.class.getName());
  final IndexDeletionPolicy delPolicy;
  if (info != null) {
    delPolicy = createInstance(info.className, IndexDeletionPolicy.class, "Deletion Policy for SOLR", this, getResourceLoader());
    if (delPolicy instanceof NamedListInitializedPlugin) {
      ((NamedListInitializedPlugin) delPolicy).init(info.initArgs);
    }
  } else {
    delPolicy = new SolrDeletionPolicy();
  }

  return new IndexDeletionPolicyWrapper(delPolicy, snapshotMgr);
}
 
Example #3
Source File: DeletionPolicyModule.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
protected void configure() {
    bind(IndexDeletionPolicy.class)
            .annotatedWith(Names.named("actual"))
            .to(KeepOnlyLastDeletionPolicy.class)
            .asEagerSingleton();

    bind(SnapshotDeletionPolicy.class)
            .asEagerSingleton();
}
 
Example #4
Source File: CreateIndexTask.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public static IndexDeletionPolicy getIndexDeletionPolicy(Config config) {
  String deletionPolicyName = config.get("deletion.policy", "org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy");
  if (deletionPolicyName.equals(NoDeletionPolicy.class.getName())) {
    return NoDeletionPolicy.INSTANCE;
  } else {
    try {
      return Class.forName(deletionPolicyName).asSubclass(IndexDeletionPolicy.class).getConstructor().newInstance();
    } catch (Exception e) {
      throw new RuntimeException("unable to instantiate class '" + deletionPolicyName + "' as IndexDeletionPolicy", e);
    }
  }
}
 
Example #5
Source File: IndexRevision.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor over the given {@link IndexWriter}. Uses the last
 * {@link IndexCommit} found in the {@link Directory} managed by the given
 * writer.
 */
public IndexRevision(IndexWriter writer) throws IOException {
  IndexDeletionPolicy delPolicy = writer.getConfig().getIndexDeletionPolicy();
  if (!(delPolicy instanceof SnapshotDeletionPolicy)) {
    throw new IllegalArgumentException("IndexWriter must be created with SnapshotDeletionPolicy");
  }
  this.writer = writer;
  this.sdp = (SnapshotDeletionPolicy) delPolicy;
  this.commit = sdp.snapshot();
  this.version = revisionVersion(commit);
  this.sourceFiles = revisionFiles(commit);
}
 
Example #6
Source File: SolrSnapshotManager.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * This method deletes index files of the {@linkplain IndexCommit} for the specified generation number.
 *
 * @param core The Solr core
 * @param dir The index directory storing the snapshot.
 * @throws IOException in case of I/O errors.
 */

@SuppressWarnings({"try", "unused"})
private static void deleteSnapshotIndexFiles(SolrCore core, Directory dir, IndexDeletionPolicy delPolicy) throws IOException {
  IndexWriterConfig conf = core.getSolrConfig().indexConfig.toIndexWriterConfig(core);
  conf.setOpenMode(OpenMode.APPEND);
  conf.setMergePolicy(NoMergePolicy.INSTANCE);//Don't want to merge any commits here!
  conf.setIndexDeletionPolicy(delPolicy);
  conf.setCodec(core.getCodec());
  try (SolrIndexWriter iw = new SolrIndexWriter("SolrSnapshotCleaner", dir, conf)) {
    // Do nothing. The only purpose of opening index writer is to invoke the Lucene IndexDeletionPolicy#onInit
    // method so that we can cleanup the files associated with specified index commit.
    // Note the index writer creates a new commit during the close() operation (which is harmless).
  }
}
 
Example #7
Source File: TestMixedDirectory.java    From RDFS with Apache License 2.0 5 votes vote down vote up
public void updateIndex(Directory dir, int base, int numDocs,
    IndexDeletionPolicy policy) throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, new StandardAnalyzer(), policy);
  writer.setMaxBufferedDocs(maxBufferedDocs);
  writer.setMergeFactor(1000);
  for (int i = 0; i < numDocs; i++) {
    addDoc(writer, base + i);
  }
  writer.close();
}
 
Example #8
Source File: TestMixedDirectory.java    From hadoop-gpu with Apache License 2.0 5 votes vote down vote up
public void updateIndex(Directory dir, int base, int numDocs,
    IndexDeletionPolicy policy) throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, new StandardAnalyzer(), policy);
  writer.setMaxBufferedDocs(maxBufferedDocs);
  writer.setMergeFactor(1000);
  for (int i = 0; i < numDocs; i++) {
    addDoc(writer, base + i);
  }
  writer.close();
}
 
Example #9
Source File: SnapshotDeletionPolicy.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * Constructs a new snapshot deletion policy that wraps the provided deletion policy.
 */
@Inject
public SnapshotDeletionPolicy(@Named("actual") IndexDeletionPolicy primary) {
    super(((IndexShardComponent) primary).shardId(), ((IndexShardComponent) primary).indexSettings());
    this.primary = primary;
}
 
Example #10
Source File: SnapshotDeletionPolicy.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public IndexDeletionPolicy clone() {
   // Lucene IW makes a clone internally but since we hold on to this instance 
   // the clone will just be the identity. See InternalEngine recovery why we need this.
   return this;
}
 
Example #11
Source File: SolrIndexWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private SolrIndexWriter(SolrCore core, String name, String path, Directory directory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec) throws IOException {
  super(directory,
        config.toIndexWriterConfig(core).
        setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND).
        setIndexDeletionPolicy(delPolicy).setCodec(codec)
        );
  log.debug("Opened Writer {}", name);
  this.name = name;
  infoStream = getConfig().getInfoStream();
  this.directory = directory;
  numOpens.incrementAndGet();
  solrMetricsContext = core.getSolrMetricsContext().getChildContext(this);
  if (config.metricsInfo != null && config.metricsInfo.initArgs != null) {
    Object v = config.metricsInfo.initArgs.get("majorMergeDocs");
    if (v != null) {
      try {
        majorMergeDocs = Long.parseLong(String.valueOf(v));
      } catch (Exception e) {
        log.warn("Invalid 'majorMergeDocs' argument, using default 512k", e);
      }
    }
    Boolean Totals = config.metricsInfo.initArgs.getBooleanArg("merge");
    Boolean Details = config.metricsInfo.initArgs.getBooleanArg("mergeDetails");
    if (Details != null) {
      mergeDetails = Details;
    } else {
      mergeDetails = false;
    }
    if (Totals != null) {
      mergeTotals = Totals;
    } else {
      mergeTotals = false;
    }
    if (mergeDetails) {
      mergeTotals = true; // override
      majorMergedDocs = solrMetricsContext.meter("docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
      majorDeletedDocs = solrMetricsContext.meter("deletedDocs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
    }
    if (mergeTotals) {
      minorMerge = solrMetricsContext.timer("minor", SolrInfoBean.Category.INDEX.toString(), "merge");
      majorMerge = solrMetricsContext.timer("major", SolrInfoBean.Category.INDEX.toString(), "merge");
      mergeErrors = solrMetricsContext.counter("errors", SolrInfoBean.Category.INDEX.toString(), "merge");
      String tag = core.getMetricTag();
      solrMetricsContext.gauge(() -> runningMajorMerges.get(), true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
      solrMetricsContext.gauge(() -> runningMinorMerges.get(), true, "running", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
      solrMetricsContext.gauge(() -> runningMajorMergesDocs.get(), true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
      solrMetricsContext.gauge(() -> runningMinorMergesDocs.get(), true, "running.docs", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
      solrMetricsContext.gauge(() -> runningMajorMergesSegments.get(), true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "major");
      solrMetricsContext.gauge(() -> runningMinorMergesSegments.get(), true, "running.segments", SolrInfoBean.Category.INDEX.toString(), "merge", "minor");
      flushMeter = solrMetricsContext.meter("flush", SolrInfoBean.Category.INDEX.toString());
    }
  }
}
 
Example #12
Source File: IndexDeletionPolicyWrapper.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public IndexDeletionPolicyWrapper(IndexDeletionPolicy deletionPolicy, SolrSnapshotMetaDataManager snapshotMgr) {
  this.deletionPolicy = deletionPolicy;
  this.snapshotMgr = snapshotMgr;
}
 
Example #13
Source File: IndexDeletionPolicyWrapper.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public IndexDeletionPolicy getWrappedDeletionPolicy() {
  return deletionPolicy;
}
 
Example #14
Source File: TableContext.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up
public IndexDeletionPolicy getIndexDeletionPolicy() {
  return _indexDeletionPolicy;
}
 
Example #15
Source File: IndexDeletionPolicyReader.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up
public IndexDeletionPolicyReader(IndexDeletionPolicy base) {
  _base = base;
}