Java Code Examples for org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy

The following examples show how to use org.apache.lucene.index.KeepOnlyLastCommitDeletionPolicy. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: RDFS   Source File: TestMixedDirectory.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 
Example 2
Source Project: hadoop-gpu   Source File: TestMixedDirectory.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedDirectoryAndPolicy() throws IOException {
  Directory readDir = new RAMDirectory();
  updateIndex(readDir, 0, numDocsPerUpdate,
      new KeepOnlyLastCommitDeletionPolicy());

  verify(readDir, numDocsPerUpdate);

  IndexOutput out =
      readDir.createOutput("_" + (numDocsPerUpdate / maxBufferedDocs + 2)
          + ".cfs");
  out.writeInt(0);
  out.close();

  Directory writeDir = new RAMDirectory();
  Directory mixedDir = new MixedDirectory(readDir, writeDir);
  updateIndex(mixedDir, numDocsPerUpdate, numDocsPerUpdate,
      new MixedDeletionPolicy());

  verify(readDir, numDocsPerUpdate);
  verify(mixedDir, 2 * numDocsPerUpdate);
}
 
Example 3
Source Project: linden   Source File: ShardWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param conf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir, Configuration conf)
    throws IOException {
  logger.info("Construct a shard writer");

  this.conf = conf;
  this.fs = fs;
  localFs = FileSystem.getLocal(conf);
  perm = new Path(shard.getDirectory());
  taxoPerm = new Path(shard.getDirectory() + ".taxonomy");
  String indexDir = tempDir + "/" + "index";
  String taxoDir = tempDir + "/" + "taxo";
  temp = new Path(indexDir);
  taxoTemp = new Path(taxoDir);

  if (localFs.exists(temp)) {
    File tempFile = new File(temp.getName());
    if (tempFile.exists()) {
      LindenReducer.deleteDir(tempFile);
    }
  }

  if (!fs.exists(perm)) {
    fs.mkdirs(perm);
  } else {
    moveToTrash(conf, perm);
    fs.mkdirs(perm);
  }

  if (!fs.exists(taxoPerm)) {
    fs.mkdirs(taxoPerm);
  } else {
    moveToTrash(conf, taxoPerm);
    fs.mkdirs(taxoPerm);
  }
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(FSDirectory.open(new File(indexDir)), config);
  taxoWriter = new DirectoryTaxonomyWriter(FSDirectory.open(new File(taxoDir)));
}
 
Example 4
Source Project: lucene-solr   Source File: IndexRevisionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNoSnapshotDeletionPolicy() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  IndexWriter writer = new IndexWriter(dir, conf);
  // should fail when IndexDeletionPolicy is not Snapshot
  expectThrows(IllegalArgumentException.class, () -> {
    new IndexRevision(writer);
  });

  writer.close();
  IOUtils.close(dir);
}
 
Example 5
Source Project: RDFS   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
 
Example 6
Source Project: RDFS   Source File: ShardWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param iconf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir,
    IndexUpdateConfiguration iconf) throws IOException {
  LOG.info("Construct a shard writer");

  this.fs = fs;
  localFs = FileSystem.getLocal(iconf.getConfiguration());
  perm = new Path(shard.getDirectory());
  temp = new Path(tempDir);

  long initGeneration = shard.getGeneration();
  if (!fs.exists(perm)) {
    assert (initGeneration < 0);
    fs.mkdirs(perm);
  } else {
    restoreGeneration(fs, perm, initGeneration);
  }
  dir =
      new MixedDirectory(fs, perm, localFs, fs.startLocalOutput(perm, temp),
          iconf.getConfiguration());

  // analyzer is null because we only use addIndexes, not addDocument
  writer =
      new IndexWriter(dir, false, null,
          initGeneration < 0 ? new KeepOnlyLastCommitDeletionPolicy()
              : new MixedDeletionPolicy());
  setParameters(iconf);
}
 
Example 7
Source Project: hadoop-gpu   Source File: IntermediateForm.java    License: Apache License 2.0 5 votes vote down vote up
private IndexWriter createWriter() throws IOException {
  IndexWriter writer =
      new IndexWriter(dir, false, null,
          new KeepOnlyLastCommitDeletionPolicy());
  writer.setUseCompoundFile(false);

  if (iconf != null) {
    int maxFieldLength = iconf.getIndexMaxFieldLength();
    if (maxFieldLength > 0) {
      writer.setMaxFieldLength(maxFieldLength);
    }
  }

  return writer;
}
 
Example 8
Source Project: hadoop-gpu   Source File: ShardWriter.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructor
 * @param fs
 * @param shard
 * @param tempDir
 * @param iconf
 * @throws IOException
 */
public ShardWriter(FileSystem fs, Shard shard, String tempDir,
    IndexUpdateConfiguration iconf) throws IOException {
  LOG.info("Construct a shard writer");

  this.fs = fs;
  localFs = FileSystem.getLocal(iconf.getConfiguration());
  perm = new Path(shard.getDirectory());
  temp = new Path(tempDir);

  long initGeneration = shard.getGeneration();
  if (!fs.exists(perm)) {
    assert (initGeneration < 0);
    fs.mkdirs(perm);
  } else {
    restoreGeneration(fs, perm, initGeneration);
  }
  dir =
      new MixedDirectory(fs, perm, localFs, fs.startLocalOutput(perm, temp),
          iconf.getConfiguration());

  // analyzer is null because we only use addIndexes, not addDocument
  writer =
      new IndexWriter(dir, false, null,
          initGeneration < 0 ? new KeepOnlyLastCommitDeletionPolicy()
              : new MixedDeletionPolicy());
  setParameters(iconf);
}
 
Example 9
Source Project: linden   Source File: IntermediateForm.java    License: Apache License 2.0 4 votes vote down vote up
private void createWriter() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LATEST, null);
  config.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
  writer = new IndexWriter(dir, config);
  taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
}
 
Example 10
@Test
public void testCreateIndex() throws IOException {
  long s = System.nanoTime();
  IndexWriterConfig conf = new IndexWriterConfig(LuceneVersionConstant.LUCENE_VERSION, new KeywordAnalyzer());
  IndexDeletionPolicyReader indexDeletionPolicy = new IndexDeletionPolicyReader(
      new KeepOnlyLastCommitDeletionPolicy());
  conf.setIndexDeletionPolicy(indexDeletionPolicy);
  FSDirectory control = FSDirectory.open(fileControl);
  Directory dir = getControlDir(control, directory);
  // The serial merge scheduler can be useful for debugging.
  // conf.setMergeScheduler(new SerialMergeScheduler());
  IndexWriter writer = new IndexWriter(dir, conf);
  int numDocs = 1000;
  DirectoryReader reader = null;
  long gen = 0;
  for (int i = 0; i < 100; i++) {
    if (reader == null) {
      reader = DirectoryReader.open(writer, true);
      gen = reader.getIndexCommit().getGeneration();
      indexDeletionPolicy.register(gen);
    } else {
      DirectoryReader old = reader;
      reader = DirectoryReader.openIfChanged(old, writer, true);
      if (reader == null) {
        reader = old;
      } else {
        long newGen = reader.getIndexCommit().getGeneration();
        indexDeletionPolicy.register(newGen);
        indexDeletionPolicy.unregister(gen);
        old.close();
        gen = newGen;
      }
    }
    assertEquals(i * numDocs, reader.numDocs());
    IndexSearcher searcher = new IndexSearcher(reader);
    NumericRangeQuery<Integer> query = NumericRangeQuery.newIntRange("id", 42, 42, true, true);
    TopDocs topDocs = searcher.search(query, 10);
    assertEquals(i, topDocs.totalHits);
    addDocuments(writer, numDocs);
  }
  writer.close(false);
  reader.close();
  long e = System.nanoTime();
  System.out.println("Total time [" + (e - s) / 1000000.0 + " ms]");
}