org.apache.lucene.search.SearcherFactory Java Examples

The following examples show how to use org.apache.lucene.search.SearcherFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestSoftDeletesRetentionMergePolicy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSoftDeleteWithTryUpdateDocValue() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_delete")
      .setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, newLogMergePolicy()));
  IndexWriter writer = new IndexWriter(dir, config);
  SearcherManager sm = new SearcherManager(writer, new SearcherFactory());
  Document d = new Document();
  d.add(new StringField("id", "0", Field.Store.YES));
  writer.addDocument(d);
  sm.maybeRefreshBlocking();
  doUpdate(new Term("id", "0"), writer,
      new NumericDocValuesField("soft_delete", 1), new NumericDocValuesField("other-field", 1));
  sm.maybeRefreshBlocking();
  assertEquals(1, writer.cloneSegmentInfos().size());
  SegmentCommitInfo si = writer.cloneSegmentInfos().info(0);
  assertEquals(1, si.getSoftDelCount());
  assertEquals(1, si.info.maxDoc());
  IOUtils.close(sm, writer, dir);
}
 
Example #2
Source File: SimpleReplicaNode.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public SimpleReplicaNode(Random random, int id, int tcpPort, Path indexPath, long curPrimaryGen, int primaryTCPPort,
                         SearcherFactory searcherFactory, boolean doCheckIndexOnClose) throws IOException {
  super(id, getDirectory(random, id, indexPath, doCheckIndexOnClose), searcherFactory, System.out);
  this.tcpPort = tcpPort;
  this.random = new Random(random.nextLong());

  // Random IO throttling on file copies: 5 - 20 MB/sec:
  double mbPerSec = 5 * (1.0 + 3*random.nextDouble());
  message(String.format(Locale.ROOT, "top: will rate limit file fetch to %.2f MB/sec", mbPerSec));
  fetchRateLimiter = new RateLimiter.SimpleRateLimiter(mbPerSec);
  this.curPrimaryTCPPort = primaryTCPPort;
  
  start(curPrimaryGen);

  // Handles fetching files from primary:
  jobs = new Jobs(this);
  jobs.setName("R" + id + ".copyJobs");
  jobs.setDaemon(true);
  jobs.start();
}
 
Example #3
Source File: TestSearcherTaxonomyManager.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private SearcherTaxonomyManager getSearcherTaxonomyManager(Directory indexDir, Directory taxoDir, SearcherFactory searcherFactory) throws IOException {
  if (random().nextBoolean()) {
    return new SearcherTaxonomyManager(indexDir, taxoDir, searcherFactory);
  } else {
    IndexReader reader = DirectoryReader.open(indexDir);
    DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
    return new SearcherTaxonomyManager(reader, taxoReader, searcherFactory);
  }
}
 
Example #4
Source File: InternalEngine.java    From crate with Apache License 2.0 5 votes vote down vote up
ExternalSearcherManager(SearcherManager internalSearcherManager, SearcherFactory searcherFactory) throws IOException {
    IndexSearcher acquire = internalSearcherManager.acquire();
    try {
        IndexReader indexReader = acquire.getIndexReader();
        assert indexReader instanceof ElasticsearchDirectoryReader :
            "searcher's IndexReader should be an ElasticsearchDirectoryReader, but got " + indexReader;
        indexReader.incRef(); // steal the reader - getSearcher will decrement if it fails
        current = SearcherManager.getSearcher(searcherFactory, indexReader, null);
    } finally {
        internalSearcherManager.release(acquire);
    }
    this.searcherFactory = searcherFactory;
    this.internalSearcherManager = internalSearcherManager;
}
 
Example #5
Source File: test.java    From vscode-extension with MIT License 5 votes vote down vote up
ExternalSearcherManager(SearcherManager internalSearcherManager, SearcherFactory searcherFactory) throws IOException {
    IndexSearcher acquire = internalSearcherManager.acquire();
    try {
        IndexReader indexReader = acquire.getIndexReader();
        assert indexReader instanceof ElasticsearchDirectoryReader:
            "searcher's IndexReader should be an ElasticsearchDirectoryReader, but got " + indexReader;
        indexReader.incRef(); // steal the reader - getSearcher will decrement if it fails
        current = SearcherManager.getSearcher(searcherFactory, indexReader, null);
    } finally {
        internalSearcherManager.release(acquire);
    }
    this.searcherFactory = searcherFactory;
    this.internalSearcherManager = internalSearcherManager;
}
 
Example #6
Source File: SimplePrimaryNode.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public SimplePrimaryNode(Random random, Path indexPath, int id, int tcpPort, long primaryGen, long forcePrimaryVersion, SearcherFactory searcherFactory,
                         boolean doFlipBitsDuringCopy, boolean doCheckIndexOnClose) throws IOException {
  super(initWriter(id, random, indexPath, doCheckIndexOnClose), id, primaryGen, forcePrimaryVersion, searcherFactory, System.out);
  this.tcpPort = tcpPort;
  this.random = new Random(random.nextLong());
  this.doFlipBitsDuringCopy = doFlipBitsDuringCopy;
}
 
Example #7
Source File: ReplicaNode.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public ReplicaNode(int id, Directory dir, SearcherFactory searcherFactory, PrintStream printStream) throws IOException {
  super(id, dir, searcherFactory, printStream);

  if (dir.getPendingDeletions().isEmpty() == false) {
    throw new IllegalArgumentException("Directory " + dir + " still has pending deleted files; cannot initialize IndexWriter");
  }

  boolean success = false;

  try {
    message("top: init replica dir=" + dir);

    // Obtain a write lock on this index since we "act like" an IndexWriter, to prevent any other IndexWriter or ReplicaNode from using it:
    writeFileLock = dir.obtainLock(IndexWriter.WRITE_LOCK_NAME);
    
    state = "init";
    deleter = new ReplicaFileDeleter(this, dir);
    success = true;
  } catch (Throwable t) {
    message("exc on init:");
    t.printStackTrace(printStream);
    throw t;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(this);
    }
  }
}
 
Example #8
Source File: ShadowEngine.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public ShadowEngine(EngineConfig engineConfig)  {
    super(engineConfig);
    SearcherFactory searcherFactory = new EngineSearcherFactory(engineConfig);
    final long nonexistentRetryTime = engineConfig.getIndexSettings()
            .getAsTime(NONEXISTENT_INDEX_RETRY_WAIT, DEFAULT_NONEXISTENT_INDEX_RETRY_WAIT)
            .getMillis();
    try {
        DirectoryReader reader = null;
        store.incRef();
        boolean success = false;
        try {
            if (Lucene.waitForIndex(store.directory(), nonexistentRetryTime)) {
                reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(store.directory()), shardId);
                this.searcherManager = new SearcherManager(reader, searcherFactory);
                this.lastCommittedSegmentInfos = readLastCommittedSegmentInfos(searcherManager, store);
                success = true;
            } else {
                throw new IllegalStateException("failed to open a shadow engine after" +
                        nonexistentRetryTime + "ms, " +
                        "directory is not an index");
            }
        } catch (Throwable e) {
            logger.warn("failed to create new reader", e);
            throw e;
        } finally {
            if (success == false) {
                IOUtils.closeWhileHandlingException(reader);
                store.decRef();
            }
        }
    } catch (IOException ex) {
        throw new EngineCreationFailureException(shardId, "failed to open index reader", ex);
    }
    logger.trace("created new ShadowEngine");
}
 
Example #9
Source File: Node.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public Node(int id, Directory dir, SearcherFactory searcherFactory, PrintStream printStream) {
  this.id = id;
  this.dir = dir;
  this.searcherFactory = searcherFactory;
  this.printStream = printStream;
}
 
Example #10
Source File: SearcherTaxonomyManager.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Creates near-real-time searcher and taxonomy reader
 *  from the corresponding writers. */
public SearcherTaxonomyManager(IndexWriter writer, SearcherFactory searcherFactory, DirectoryTaxonomyWriter taxoWriter) throws IOException {
  this(writer, true, searcherFactory, taxoWriter);
}
 
Example #11
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testRandomOperations() throws Exception {
  IndexWriterConfig iwc = newIndexWriterConfig();
  iwc.setMergePolicy(new FilterMergePolicy(newMergePolicy()) {
    boolean keepFullyDeletedSegment = random().nextBoolean();

    @Override
    public boolean keepFullyDeletedSegment(IOSupplier<CodecReader> readerIOSupplier) {
      return keepFullyDeletedSegment;
    }
  });
  try (Directory dir = newDirectory();
       IndexWriter writer = new IndexWriter(dir, iwc);
       SearcherManager sm = new SearcherManager(writer, new SearcherFactory())) {
    Semaphore numOperations = new Semaphore(10 + random().nextInt(1000));
    boolean singleDoc = random().nextBoolean();
    Thread[] threads = new Thread[1 + random().nextInt(4)];
    CountDownLatch latch = new CountDownLatch(threads.length);
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new Thread(() -> {
        latch.countDown();
        try {
          latch.await();
          while (numOperations.tryAcquire()) {
            String id = singleDoc ? "1" : Integer.toString(random().nextInt(10));
            Document doc = new Document();
            doc.add(new StringField("id", id, Field.Store.YES));
            if (random().nextInt(10) <= 2) {
              writer.updateDocument(new Term("id", id), doc);
            } else if (random().nextInt(10) <= 2) {
              writer.deleteDocuments(new Term("id", id));
            } else {
              writer.addDocument(doc);
            }
            if (random().nextInt(100) < 10) {
              sm.maybeRefreshBlocking();
            }
            if (random().nextInt(100) < 5) {
              writer.commit();
            }
            if (random().nextInt(100) < 1) {
              writer.forceMerge(1 + random().nextInt(10), random().nextBoolean());
            }
          }
        } catch (Exception e) {
          throw new AssertionError(e);
        }
      });
      threads[i].start();
    }
    for (Thread thread : threads) {
      thread.join();
    }
  }
}
 
Example #12
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testRandomOperationsWithSoftDeletes() throws Exception {
  IndexWriterConfig iwc = newIndexWriterConfig();
  AtomicInteger seqNo = new AtomicInteger(-1);
  AtomicInteger retainingSeqNo = new AtomicInteger();
  iwc.setSoftDeletesField("soft_deletes");
  iwc.setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_deletes",
      () -> LongPoint.newRangeQuery("seq_no", retainingSeqNo.longValue(), Long.MAX_VALUE), newMergePolicy()));
  try (Directory dir = newDirectory();
       IndexWriter writer = new IndexWriter(dir, iwc);
       SearcherManager sm = new SearcherManager(writer, new SearcherFactory())) {
    Semaphore numOperations = new Semaphore(10 + random().nextInt(1000));
    boolean singleDoc = random().nextBoolean();
    Thread[] threads = new Thread[1 + random().nextInt(4)];
    CountDownLatch latch = new CountDownLatch(threads.length);
    for (int i = 0; i < threads.length; i++) {
      threads[i] = new Thread(() -> {
        latch.countDown();
        try {
          latch.await();
          while (numOperations.tryAcquire()) {
            String id = singleDoc ? "1" : Integer.toString(random().nextInt(10));
            Document doc = new Document();
            doc.add(new StringField("id", id, Field.Store.YES));
            doc.add(new LongPoint("seq_no", seqNo.getAndIncrement()));
            if (random().nextInt(10) <= 2) {
              if (random().nextBoolean()) {
                doc.add(new NumericDocValuesField(iwc.softDeletesField, 1));
              }
              writer.softUpdateDocument(new Term("id", id), doc, new NumericDocValuesField(iwc.softDeletesField, 1));
            } else {
              writer.addDocument(doc);
            }
            if (random().nextInt(100) < 10) {
              int min = retainingSeqNo.get();
              int max = seqNo.get();
              if (min < max && random().nextBoolean()) {
                retainingSeqNo.compareAndSet(min, min - random().nextInt(max - min));
              }
            }
            if (random().nextInt(100) < 10) {
              sm.maybeRefreshBlocking();
            }
            if (random().nextInt(100) < 5) {
              writer.commit();
            }
            if (random().nextInt(100) < 1) {
              writer.forceMerge(1 + random().nextInt(10), random().nextBoolean());
            }
          }
        } catch (Exception e) {
          throw new AssertionError(e);
        }
      });
      threads[i].start();
    }
    for (Thread thread : threads) {
      thread.join();
    }
  }
}
 
Example #13
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testMaxCompletedSequenceNumber() throws IOException, InterruptedException {
  try (Directory dir = newDirectory();
       IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());) {
    assertEquals(1, writer.addDocument(new Document()));
    assertEquals(2, writer.updateDocument(new Term("foo", "bar"), new Document()));
    writer.flushNextBuffer();
    assertEquals(3, writer.commit());
    assertEquals(4, writer.addDocument(new Document()));
    assertEquals(4, writer.getMaxCompletedSequenceNumber());
    // commit moves seqNo by 2 since there is one DWPT that could still be in-flight
    assertEquals(6, writer.commit());
    assertEquals(6, writer.getMaxCompletedSequenceNumber());
    assertEquals(7, writer.addDocument(new Document()));
    writer.getReader().close();
    // getReader moves seqNo by 2 since there is one DWPT that could still be in-flight
    assertEquals(9, writer.getMaxCompletedSequenceNumber());
  }
  try (Directory dir = newDirectory();
       IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
       SearcherManager manager = new SearcherManager(writer, new SearcherFactory())) {
    CountDownLatch start = new CountDownLatch(1);
    int numDocs = 100 + random().nextInt(500);
    AtomicLong maxCompletedSeqID = new AtomicLong(-1);
    Thread[] threads = new Thread[2 + random().nextInt(2)];
    for (int i = 0; i < threads.length; i++) {
      int idx = i;
      threads[i] = new Thread(() -> {
        try {
          start.await();
          for (int j = 0; j < numDocs; j++) {
            Document doc = new Document();
            String id = idx +"-"+j;
            doc.add(new StringField("id", id, Field.Store.NO));
            long seqNo = writer.addDocument(doc);
            if (maxCompletedSeqID.get() < seqNo) {
              long maxCompletedSequenceNumber = writer.getMaxCompletedSequenceNumber();
              manager.maybeRefreshBlocking();
              maxCompletedSeqID.updateAndGet(oldVal-> Math.max(oldVal, maxCompletedSequenceNumber));
            }
            IndexSearcher acquire = manager.acquire();
            try {
              assertEquals(1, acquire.search(new TermQuery(new Term("id", id)), 10).totalHits.value);
            } finally {
              manager.release(acquire);
            }
          }
        } catch (Exception e) {
          throw new AssertionError(e);
        }
      });
      threads[i].start();
    }
    start.countDown();
    for (int i = 0; i < threads.length; i++) {
      threads[i].join();
    }
  }
}
 
Example #14
Source File: TestTryDelete.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testTryDeleteDocument ()
  throws IOException
{
  Directory directory = createIndex();

  IndexWriter writer = getWriter(directory);

  ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer,
                                                            new SearcherFactory());

  IndexSearcher searcher = mgr.acquire();

  TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")),
                                    100);
  assertEquals(1, topDocs.totalHits.value);

  long result;
  if (random().nextBoolean()) {
    IndexReader r = DirectoryReader.open(writer);
    result = writer.tryDeleteDocument(r, 0);
    r.close();
  } else {
    result = writer.tryDeleteDocument(searcher.getIndexReader(), 0);
  }

  // The tryDeleteDocument should have succeeded:
  assertTrue(result != -1);

  assertTrue(writer.hasDeletions());

  if (random().nextBoolean()) {
    writer.commit();
  }

  assertTrue(writer.hasDeletions());

  mgr.maybeRefresh();

  searcher = mgr.acquire();

  topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100);

  assertEquals(0, topDocs.totalHits.value);
}
 
Example #15
Source File: PrimaryNode.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public PrimaryNode(IndexWriter writer, int id, long primaryGen, long forcePrimaryVersion,
                   SearcherFactory searcherFactory, PrintStream printStream) throws IOException {
  super(id, writer.getDirectory(), searcherFactory, printStream);
  message("top: now init primary");
  this.writer = writer;
  this.primaryGen = primaryGen;

  try {
    // So that when primary node's IndexWriter finishes a merge, but before it cuts over to the merged segment,
    // it copies it out to the replicas.  This ensures the whole system's NRT latency remains low even when a
    // large merge completes:
    writer.getConfig().setMergedSegmentWarmer(new PreCopyMergedSegmentWarmer(this));

    message("IWC:\n" + writer.getConfig());
    message("dir:\n" + writer.getDirectory());
    message("commitData: " + writer.getLiveCommitData());

    // Record our primaryGen in the userData, and set initial version to 0:
    Map<String,String> commitData = new HashMap<>();
    Iterable<Map.Entry<String,String>> iter = writer.getLiveCommitData();
    if (iter != null) {
      for(Map.Entry<String,String> ent : iter) {
        commitData.put(ent.getKey(), ent.getValue());
      }
    }
    commitData.put(PRIMARY_GEN_KEY, Long.toString(primaryGen));
    if (commitData.get(VERSION_KEY) == null) {
      commitData.put(VERSION_KEY, "0");
      message("add initial commitData version=0");
    } else {
      message("keep current commitData version=" + commitData.get(VERSION_KEY));
    }
    writer.setLiveCommitData(commitData.entrySet(), false);

    // We forcefully advance the SIS version to an unused future version.  This is necessary if the previous primary crashed and we are
    // starting up on an "older" index, else versions can be illegally reused but show different results:
    if (forcePrimaryVersion != -1) {
      message("now forcePrimaryVersion to version=" + forcePrimaryVersion);
      writer.advanceSegmentInfosVersion(forcePrimaryVersion);
    }

    mgr = new SearcherManager(writer, true, true, searcherFactory);
    setCurrentInfos(Collections.<String>emptySet());
    message("init: infos version=" + curInfos.getVersion());

  } catch (Throwable t) {
    message("init: exception");
    t.printStackTrace(printStream);
    throw new RuntimeException(t);
  }
}
 
Example #16
Source File: Searcher.java    From sql-layer with GNU Affero General Public License v3.0 4 votes vote down vote up
public Searcher(FullTextIndexShared index, Analyzer analyzer) throws IOException {
    this.index = index;
    this.searcherManager = new SearcherManager(index.open(), new SearcherFactory());
}
 
Example #17
Source File: TestTryDelete.java    From lucene-solr with Apache License 2.0 3 votes vote down vote up
public void testDeleteDocuments ()
  throws IOException
{
  Directory directory = createIndex();

  IndexWriter writer = getWriter(directory);

  ReferenceManager<IndexSearcher> mgr = new SearcherManager(writer,
                                                            new SearcherFactory());

  IndexSearcher searcher = mgr.acquire();

  TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")),
                                    100);
  assertEquals(1, topDocs.totalHits.value);

  long result = writer.deleteDocuments(new TermQuery(new Term("foo", "0")));

  assertTrue(result != -1);

  // writer.commit();

  assertTrue(writer.hasDeletions());

  mgr.maybeRefresh();

  searcher = mgr.acquire();

  topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100);

  assertEquals(0, topDocs.totalHits.value);
}