Java Code Examples for org.apache.lucene.index.IndexReader#open()

private IndexReader getIndexReader(String indexName, String locale)
        throws CorruptIndexException, IOException {
    String path = "";
    if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) {
        path = indexWorkDir + File.separator +
            getDocIndexPath(locale);
    }
    else {
        path = indexWorkDir + indexName;
    }
    log.info("IndexManager::getIndexReader(" + indexName + ", " + locale +
            ") path = " + path);
    File f = new File(path);
    IndexReader retval = IndexReader.open(FSDirectory.getDirectory(f));
    return retval;
}
 

private IndexReader buildAndRegisterDeltaReader(String id) throws IOException
{
    IndexReader reader;
    // only register on write to avoid any locking for transactions that only ever read
    File location = getDeltaLocation(id);
    // File location = ensureDeltaIsRegistered(id);
    // Create a dummy index reader to deal with empty indexes and not
    // persist these.
    if (IndexReader.indexExists(location))
    {
        reader = IndexReader.open(location);
    }
    else
    {
        reader = IndexReader.open(emptyIndex);
    }
    return reader;
}
 

@Test
public void testSingleTerm() throws IOException {
    RAMDirectory d = new RAMDirectory();
    IndexWriter w = new IndexWriter(d, null, true, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    doc.add(new Field("int", "1", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
    w.addDocument(doc);
    w.close();

    IndexReader r = IndexReader.open(d);
    LuceneUnsortedIntTermDocIterator iter = LuceneUnsortedIntTermDocIterator.create(r, "int");
    assertTrue(iter.nextTerm());
    assertEquals(1, iter.term());
    int[] docs = new int[2];
    assertEquals(1, iter.nextDocs(docs));
    assertEquals(0, docs[0]);
    assertFalse(iter.nextTerm());
    r.close();
}
 

public OfflineSearcher(File indexDir,File rootOfIndexedFiles) {
    try {
        this.rootOfIndexedFiles = rootOfIndexedFiles;
        log.info("indexDir is '" + indexDir + "'");
        if (rootOfIndexedFiles!=null) {
            log.info("paths to file names in the index are relative to '" + rootOfIndexedFiles + "'");
        } else {
            log.info("paths to files in index are absolute");
        }
        IndexReader reader = IndexReader.open(FSDirectory.open(indexDir), true); // only searching, so read-only=true
        searcher = new IndexSearcher(reader);
        if (searcher==null) {
            log.error("can't open indexSearcher for "+indexDir);
        }
    } catch (IOException ex) {
        log.error(" caught IOException opening indexDir "+indexDir+" with message: " + ex.getMessage());
    }
}
 

private static void indexDocs(File file, String index, boolean create)
      throws Exception {
   if (!create) {				  // incrementally update

     reader = IndexReader.open(index);		  // open existing index
     uidIter = reader.terms(new Term("uid", "")); // init uid iterator

     indexDocs(file);

     if (deleting) {				  // delete rest of stale docs
while (uidIter.term() != null && uidIter.term().field() == "uid") {
  System.out.println("deleting " +
		     HTMLDocument.uid2url(uidIter.term().text()));
  reader.delete(uidIter.term());
  uidIter.next();
}
deleting = false;
     }

     uidIter.close();				  // close uid iterator
     reader.close();				  // close existing index

   } else					  // don't have exisiting
     indexDocs(file);
 }
 

public static IndexReader getReader(String path) throws IOException
{
	if (!readerMap.containsKey(path))
	{
		synchronized(Indexes.class)
		{
			if (!readerMap.containsKey(path))
			{
				log.info("["+path+"] Opening...");
				if (! new File(path).exists())
					throw new ConfigurationException("Unable to find index in "+path);

				IndexReader r = IndexReader.open(new SimpleFSDirectory(new File(path)), true);
				readerMap.put(path, r);
				log.info("["+path+"] Opened. Memory: "+ExternalSortUtils.memSize(false));
			}
		}
	}
	
	return readerMap.get(path);
}
 

private static void indexDocs(File file, String index, boolean create)
      throws Exception {
   if (!create) {				  // incrementally update

     reader = IndexReader.open(index);		  // open existing index
     uidIter = reader.terms(new Term("uid", "")); // init uid iterator

     indexDocs(file);

     if (deleting) {				  // delete rest of stale docs
while (uidIter.term() != null && uidIter.term().field() == "uid") {
  System.out.println("deleting " +
		     HTMLDocument.uid2url(uidIter.term().text()));
  reader.delete(uidIter.term());
  uidIter.next();
}
deleting = false;
     }

     uidIter.close();				  // close uid iterator
     reader.close();				  // close existing index

   } else					  // don't have exisiting
     indexDocs(file);
 }
 

private Flags retrieveFlags(Mailbox mailbox, MessageUid uid) throws IOException {
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        Flags retrievedFlags = new Flags();

        BooleanQuery query = new BooleanQuery();
        query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailbox.getMailboxId().serialize())), BooleanClause.Occur.MUST);
        query.add(createQuery(MessageRange.one(uid)), BooleanClause.Occur.MUST);
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);

        TopDocs docs = searcher.search(query, 100000);
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);

            Stream.of(doc.getValues(FLAGS_FIELD))
                .forEach(flag -> fromString(flag).ifPresentOrElse(retrievedFlags::add, () -> retrievedFlags.add(flag)));
        }
        return retrievedFlags;
    }
}
 

private void update(MailboxId mailboxId, MessageUid uid, Flags f) throws IOException {
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        BooleanQuery query = new BooleanQuery();
        query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailboxId.serialize())), BooleanClause.Occur.MUST);
        query.add(createQuery(MessageRange.one(uid)), BooleanClause.Occur.MUST);
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);

        TopDocs docs = searcher.search(query, 100000);
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);

            doc.removeFields(FLAGS_FIELD);
            indexFlags(doc, f);

            writer.updateDocument(new Term(ID_FIELD, doc.get(ID_FIELD)), doc);
        }
    }
}
 

@CheckForNull
private synchronized IndexReader getReader() throws IOException {
    if (cachedReader == null) {
        try {
            cachedReader = IndexReader.open(getDirectory(),true);
        } catch (FileNotFoundException fnf) {
            //pass - returns null
        }
    }
    return cachedReader;
}
 

public static void main(String[] args) throws IOException, InterruptedException {
    final String luceneDir = args[0];
    final String simpleDir = args[1];

    final FlamdexReader reader1 = new LuceneFlamdexReader(IndexReader.open(luceneDir));
    final FlamdexReader reader2 = SimpleFlamdexReader.open(simpleDir);

    benchmark(reader1, reader2);
}
 

private void verify(Shard[] shards) throws IOException {
  // verify the index
  IndexReader[] readers = new IndexReader[shards.length];
  for (int i = 0; i < shards.length; i++) {
    Directory dir =
        new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
            false, conf);
    readers[i] = IndexReader.open(dir);
  }

  IndexReader reader = new MultiReader(readers);
  IndexSearcher searcher = new IndexSearcher(reader);
  Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));
  assertEquals(0, hits.length());

  hits = searcher.search(new TermQuery(new Term("content", "hadoop")));
  assertEquals(numDocsPerRun / 2, hits.length());

  int[] counts = new int[numDocsPerRun];
  for (int i = 0; i < hits.length(); i++) {
    Document doc = hits.doc(i);
    counts[Integer.parseInt(doc.get("id"))]++;
  }

  for (int i = 0; i < numDocsPerRun; i++) {
    if (i % 2 == 0) {
      assertEquals(0, counts[i]);
    } else {
      assertEquals(1, counts[i]);
    }
  }

  searcher.close();
  reader.close();
}
 

private List<SearchResult> searchMultimap(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
    ImmutableList.Builder<SearchResult> results = ImmutableList.builder();

    Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
    
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        BooleanQuery query = new BooleanQuery();
        query.add(inMailboxes, BooleanClause.Occur.MUST);
        // Not return flags documents
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);

        List<Criterion> crits = searchQuery.getCriteria();
        for (Criterion crit : crits) {
            query.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
        }

        // query for all the documents sorted as specified in the SearchQuery
        TopDocs docs = searcher.search(query, null, maxQueryResults, createSort(searchQuery.getSorts()));
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);
            MessageUid uid = MessageUid.of(Long.parseLong(doc.get(UID_FIELD)));
            MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD));
            Optional<MessageId> messageId = toMessageId(Optional.ofNullable(doc.get(MESSAGE_ID_FIELD)));
            results.add(new SearchResult(messageId, mailboxId, uid));
        }
    } catch (IOException e) {
        throw new MailboxException("Unable to search the mailbox", e);
    }
    return results.build();
}
 

@Before
public void initializeLucene() throws IOException {
    directory = FSDirectory.open(new File(Main.LUCENE_INDEX));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LuceneSearch.LUCENE_VERSION, getDefaultFieldName(), new StandardAnalyzer(LuceneSearch.LUCENE_VERSION));
}
 

private void verify(Shard[] shards) throws IOException {
  // verify the index
  IndexReader[] readers = new IndexReader[shards.length];
  for (int i = 0; i < shards.length; i++) {
    Directory dir =
        new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
            false, conf);
    readers[i] = IndexReader.open(dir);
  }

  IndexReader reader = new MultiReader(readers);
  IndexSearcher searcher = new IndexSearcher(reader);
  Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));
  assertEquals(0, hits.length());

  hits = searcher.search(new TermQuery(new Term("content", "hadoop")));
  assertEquals(numDocsPerRun / 2, hits.length());

  int[] counts = new int[numDocsPerRun];
  for (int i = 0; i < hits.length(); i++) {
    Document doc = hits.doc(i);
    counts[Integer.parseInt(doc.get("id"))]++;
  }

  for (int i = 0; i < numDocsPerRun; i++) {
    if (i % 2 == 0) {
      assertEquals(0, counts[i]);
    } else {
      assertEquals(1, counts[i]);
    }
  }

  searcher.close();
  reader.close();
}
 

/**
 * Return a {@link Query} which is build based on the given {@link SearchQuery.FlagCriterion}. This is kind of a hack
 * as it will do a search for the flags in this method and
 */
private Query createFlagQuery(String flag, boolean isSet, Query inMailboxes, Collection<MessageUid> recentUids) throws MailboxException {
    BooleanQuery query = new BooleanQuery();
    
    if (isSet) {   
        query.add(new TermQuery(new Term(FLAGS_FIELD, flag)), BooleanClause.Occur.MUST);
    } else {
        // lucene does not support simple NOT queries so we do some nasty hack here
        BooleanQuery bQuery = new BooleanQuery();
        bQuery.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);
        bQuery.add(new TermQuery(new Term(FLAGS_FIELD, flag)),BooleanClause.Occur.MUST_NOT);
        
        query.add(bQuery, BooleanClause.Occur.MUST);
    }
    query.add(inMailboxes, BooleanClause.Occur.MUST);


    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        Set<MessageUid> uids = new HashSet<>();

        // query for all the documents sorted by uid
        TopDocs docs = searcher.search(query, null, maxQueryResults, new Sort(UID_SORT));
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            MessageUid uid = MessageUid.of(Long.parseLong(searcher.doc(sDoc.doc).get(UID_FIELD)));
            uids.add(uid);
        }
        
        // add or remove recent uids
        if (flag.equalsIgnoreCase("\\RECENT")) {
            if (isSet) {
                uids.addAll(recentUids);
            } else {
                uids.removeAll(recentUids);
            }
        }
        
        List<MessageRange> ranges = MessageRange.toRanges(new ArrayList<>(uids));
        UidRange[] nRanges = new UidRange[ranges.size()];
        for (int i = 0; i < ranges.size(); i++) {
            MessageRange range = ranges.get(i);
            nRanges[i] = new UidRange(range.getUidFrom(), range.getUidTo());
        }
        return createUidQuery((UidCriterion) SearchQuery.uid(nRanges));
    } catch (IOException e) {
        throw new MailboxException("Unable to search mailbox " + inMailboxes, e);
    }
}
 

/**
 * Creates a new spell-check index based on search-index
 */
public void createSpellIndex() {
    if (isSpellCheckEnabled) {
        IndexReader indexReader = null;
        try {
            log.info("Start generating Spell-Index...");
            long startSpellIndexTime = 0;
            if (log.isDebugEnabled()) {
                startSpellIndexTime = System.currentTimeMillis();
            }
            final Directory indexDir = FSDirectory.open(new File(indexPath));
            indexReader = IndexReader.open(indexDir);
            // 1. Create content spellIndex
            final File spellDictionaryFile = new File(spellDictionaryPath);
            final Directory contentSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + CONTENT_PATH));// true
            final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
            final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
            contentSpellChecker.indexDictionary(contentDictionary);
            // 2. Create title spellIndex
            final Directory titleSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + TITLE_PATH));// true
            final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
            final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
            titleSpellChecker.indexDictionary(titleDictionary);
            // 3. Create description spellIndex
            final Directory descriptionSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + DESCRIPTION_PATH));// true
            final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
            final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
            descriptionSpellChecker.indexDictionary(descriptionDictionary);
            // 4. Create author spellIndex
            final Directory authorSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + AUTHOR_PATH));// true
            final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
            final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
            authorSpellChecker.indexDictionary(authorDictionary);

            // Merge all part spell indexes (content,title etc.) to one common spell index
            final Directory spellIndexDirectory = FSDirectory.open(spellDictionaryFile);// true
            final IndexWriter merger = new IndexWriter(spellIndexDirectory, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
            final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
            merger.addIndexesNoOptimize(directories);
            merger.optimize();
            merger.close();
            spellChecker = new SpellChecker(spellIndexDirectory);
            spellChecker.setAccuracy(0.7f);
            if (log.isDebugEnabled()) {
                log.debug("SpellIndex created in " + (System.currentTimeMillis() - startSpellIndexTime) + "ms");
            }
            log.info("New generated Spell-Index ready to use.");
        } catch (final IOException ioEx) {
            log.warn("Can not create SpellIndex", ioEx);
        } finally {
            if (indexReader != null) {
                try {
                    indexReader.close();
                } catch (final IOException e) {
                    log.warn("Can not close indexReader properly", e);
                }
            }
        }
    }
}
 

@Test
    public void testCloseSessionDuringFTGS() throws ImhotepOutOfMemoryException, IOException, InterruptedException {
        String tempDir = Files.getTempDirectory("asdf", "");
        try {
            IndexWriter w = new IndexWriter(tempDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            Random rand = new Random();
            for (int i = 0; i < 1000000; ++i) {
                int numTerms = rand.nextInt(5) + 5;
                Document doc = new Document();
                for (int t = 0; t < numTerms; ++t) {
                    doc.add(new Field("sf1", Integer.toString(rand.nextInt(10000)), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                }
                w.addDocument(doc);
            }

            w.close();

            final AtomicBoolean closed = new AtomicBoolean(false);
            FlamdexReader r = new LuceneFlamdexReader(IndexReader.open(tempDir)) {
                @Override
                public void close() throws IOException {
                    super.close();
                    closed.set(true);
                }
            };
            final ExecutorService executor = Executors.newCachedThreadPool();
            try {
                ImhotepSession session =
                        new MTImhotepMultiSession(new ImhotepLocalSession[] { new ImhotepLocalSession(r) },
                                                  new MemoryReservationContext(
                                                                               new ImhotepMemoryPool(
                                                                                                     Long.MAX_VALUE)),
                                                  executor, null);
//                FTGSIterator iter = session.getFTGSIterator(new String[]{}, new String[]{"sf1"}); //TODO fix this
                session.close();
                assertTrue(closed.get());
            } finally {
                executor.shutdown();
            }
        } finally {
            Files.delete(tempDir);
        }
    }
 

private IndexReader createIndexReader(IndexType indexType) throws IOException {
    File dir = getIndexDirectory(indexType);
    return IndexReader.open(FSDirectory.open(dir), true);
}
 

/**
 * Creates the object and opens the index file.
 *
 * @param dictType
 *            the dictionary we will use for the search.
 * @param dictionaryPath
 *            overrides default dictionary location if non-null. An absolute
 *            os-specific path, e.g. /sdcard/aedict/index.
 * @param sort if true then the result list is always sorted.
 * @throws IOException
 *             on I/O error.
 */
public LuceneSearch(final DictTypeEnum dictType, final String dictionaryPath, final boolean sort) throws IOException {
    this.dictType = dictType;
    directory = FSDirectory.open(new File(dictionaryPath != null ? dictionaryPath : dictType.getDefaultDictionaryPath()));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LUCENE_VERSION, "contents", new StandardAnalyzer(LUCENE_VERSION));
    this.sort = sort;
}