Java Code Examples for org.apache.lucene.index.IndexReader#open()

The following examples show how to use org.apache.lucene.index.IndexReader#open() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: uyuni   File: IndexManager.java    License: GNU General Public License v2.0 6 votes vote down vote up
private IndexReader getIndexReader(String indexName, String locale)
        throws CorruptIndexException, IOException {
    String path = "";
    if (indexName.compareTo(BuilderFactory.DOCS_TYPE) == 0) {
        path = indexWorkDir + File.separator +
            getDocIndexPath(locale);
    }
    else {
        path = indexWorkDir + indexName;
    }
    log.info("IndexManager::getIndexReader(" + indexName + ", " + locale +
            ") path = " + path);
    File f = new File(path);
    IndexReader retval = IndexReader.open(FSDirectory.getDirectory(f));
    return retval;
}
 
Example 2
private IndexReader buildAndRegisterDeltaReader(String id) throws IOException
{
    IndexReader reader;
    // only register on write to avoid any locking for transactions that only ever read
    File location = getDeltaLocation(id);
    // File location = ensureDeltaIsRegistered(id);
    // Create a dummy index reader to deal with empty indexes and not
    // persist these.
    if (IndexReader.indexExists(location))
    {
        reader = IndexReader.open(location);
    }
    else
    {
        reader = IndexReader.open(emptyIndex);
    }
    return reader;
}
 
Example 3
Source Project: SEAL   File: OfflineSearcher.java    License: Apache License 2.0 6 votes vote down vote up
public OfflineSearcher(File indexDir,File rootOfIndexedFiles) {
    try {
        this.rootOfIndexedFiles = rootOfIndexedFiles;
        log.info("indexDir is '" + indexDir + "'");
        if (rootOfIndexedFiles!=null) {
            log.info("paths to file names in the index are relative to '" + rootOfIndexedFiles + "'");
        } else {
            log.info("paths to files in index are absolute");
        }
        IndexReader reader = IndexReader.open(FSDirectory.open(indexDir), true); // only searching, so read-only=true
        searcher = new IndexSearcher(reader);
        if (searcher==null) {
            log.error("can't open indexSearcher for "+indexDir);
        }
    } catch (IOException ex) {
        log.error(" caught IOException opening indexDir "+indexDir+" with message: " + ex.getMessage());
    }
}
 
Example 4
private static void indexDocs(File file, String index, boolean create)
      throws Exception {
   if (!create) {				  // incrementally update

     reader = IndexReader.open(index);		  // open existing index
     uidIter = reader.terms(new Term("uid", "")); // init uid iterator

     indexDocs(file);

     if (deleting) {				  // delete rest of stale docs
while (uidIter.term() != null && uidIter.term().field() == "uid") {
  System.out.println("deleting " +
		     HTMLDocument.uid2url(uidIter.term().text()));
  reader.delete(uidIter.term());
  uidIter.next();
}
deleting = false;
     }

     uidIter.close();				  // close uid iterator
     reader.close();				  // close existing index

   } else					  // don't have exisiting
     indexDocs(file);
 }
 
Example 5
Source Project: tagme   File: Indexes.java    License: Apache License 2.0 6 votes vote down vote up
public static IndexReader getReader(String path) throws IOException
{
	if (!readerMap.containsKey(path))
	{
		synchronized(Indexes.class)
		{
			if (!readerMap.containsKey(path))
			{
				log.info("["+path+"] Opening...");
				if (! new File(path).exists())
					throw new ConfigurationException("Unable to find index in "+path);

				IndexReader r = IndexReader.open(new SimpleFSDirectory(new File(path)), true);
				readerMap.put(path, r);
				log.info("["+path+"] Opened. Memory: "+ExternalSortUtils.memSize(false));
			}
		}
	}
	
	return readerMap.get(path);
}
 
Example 6
private static void indexDocs(File file, String index, boolean create)
      throws Exception {
   if (!create) {				  // incrementally update

     reader = IndexReader.open(index);		  // open existing index
     uidIter = reader.terms(new Term("uid", "")); // init uid iterator

     indexDocs(file);

     if (deleting) {				  // delete rest of stale docs
while (uidIter.term() != null && uidIter.term().field() == "uid") {
  System.out.println("deleting " +
		     HTMLDocument.uid2url(uidIter.term().text()));
  reader.delete(uidIter.term());
  uidIter.next();
}
deleting = false;
     }

     uidIter.close();				  // close uid iterator
     reader.close();				  // close existing index

   } else					  // don't have exisiting
     indexDocs(file);
 }
 
Example 7
private void update(MailboxId mailboxId, MessageUid uid, Flags f) throws IOException {
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        BooleanQuery query = new BooleanQuery();
        query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailboxId.serialize())), BooleanClause.Occur.MUST);
        query.add(createQuery(MessageRange.one(uid)), BooleanClause.Occur.MUST);
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);

        TopDocs docs = searcher.search(query, 100000);
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);

            doc.removeFields(FLAGS_FIELD);
            indexFlags(doc, f);

            writer.updateDocument(new Term(ID_FIELD, doc.get(ID_FIELD)), doc);
        }
    }
}
 
Example 8
private Flags retrieveFlags(Mailbox mailbox, MessageUid uid) throws IOException {
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        Flags retrievedFlags = new Flags();

        BooleanQuery query = new BooleanQuery();
        query.add(new TermQuery(new Term(MAILBOX_ID_FIELD, mailbox.getMailboxId().serialize())), BooleanClause.Occur.MUST);
        query.add(createQuery(MessageRange.one(uid)), BooleanClause.Occur.MUST);
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);

        TopDocs docs = searcher.search(query, 100000);
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);

            Stream.of(doc.getValues(FLAGS_FIELD))
                .forEach(flag -> fromString(flag).ifPresentOrElse(retrievedFlags::add, () -> retrievedFlags.add(flag)));
        }
        return retrievedFlags;
    }
}
 
Example 9
@Test
public void testSingleTerm() throws IOException {
    RAMDirectory d = new RAMDirectory();
    IndexWriter w = new IndexWriter(d, null, true, IndexWriter.MaxFieldLength.LIMITED);
    Document doc = new Document();
    doc.add(new Field("int", "1", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
    w.addDocument(doc);
    w.close();

    IndexReader r = IndexReader.open(d);
    LuceneUnsortedIntTermDocIterator iter = LuceneUnsortedIntTermDocIterator.create(r, "int");
    assertTrue(iter.nextTerm());
    assertEquals(1, iter.term());
    int[] docs = new int[2];
    assertEquals(1, iter.nextDocs(docs));
    assertEquals(0, docs[0]);
    assertFalse(iter.nextTerm());
    r.close();
}
 
Example 10
Source Project: netbeans   File: MemoryIndex.java    License: Apache License 2.0 5 votes vote down vote up
@CheckForNull
private synchronized IndexReader getReader() throws IOException {
    if (cachedReader == null) {
        try {
            cachedReader = IndexReader.open(getDirectory(),true);
        } catch (FileNotFoundException fnf) {
            //pass - returns null
        }
    }
    return cachedReader;
}
 
Example 11
private void verify(Shard[] shards) throws IOException {
  // verify the index
  IndexReader[] readers = new IndexReader[shards.length];
  for (int i = 0; i < shards.length; i++) {
    Directory dir =
        new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
            false, conf);
    readers[i] = IndexReader.open(dir);
  }

  IndexReader reader = new MultiReader(readers);
  IndexSearcher searcher = new IndexSearcher(reader);
  Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));
  assertEquals(0, hits.length());

  hits = searcher.search(new TermQuery(new Term("content", "hadoop")));
  assertEquals(numDocsPerRun / 2, hits.length());

  int[] counts = new int[numDocsPerRun];
  for (int i = 0; i < hits.length(); i++) {
    Document doc = hits.doc(i);
    counts[Integer.parseInt(doc.get("id"))]++;
  }

  for (int i = 0; i < numDocsPerRun; i++) {
    if (i % 2 == 0) {
      assertEquals(0, counts[i]);
    } else {
      assertEquals(1, counts[i]);
    }
  }

  searcher.close();
  reader.close();
}
 
Example 12
@Before
public void initializeLucene() throws IOException {
    directory = FSDirectory.open(new File(Main.LUCENE_INDEX));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LuceneSearch.LUCENE_VERSION, getDefaultFieldName(), new StandardAnalyzer(LuceneSearch.LUCENE_VERSION));
}
 
Example 13
private List<SearchResult> searchMultimap(Collection<MailboxId> mailboxIds, SearchQuery searchQuery) throws MailboxException {
    ImmutableList.Builder<SearchResult> results = ImmutableList.builder();

    Query inMailboxes = buildQueryFromMailboxes(mailboxIds);
    
    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        BooleanQuery query = new BooleanQuery();
        query.add(inMailboxes, BooleanClause.Occur.MUST);
        // Not return flags documents
        query.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST_NOT);

        List<Criterion> crits = searchQuery.getCriteria();
        for (Criterion crit : crits) {
            query.add(createQuery(crit, inMailboxes, searchQuery.getRecentMessageUids()), BooleanClause.Occur.MUST);
        }

        // query for all the documents sorted as specified in the SearchQuery
        TopDocs docs = searcher.search(query, null, maxQueryResults, createSort(searchQuery.getSorts()));
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            Document doc = searcher.doc(sDoc.doc);
            MessageUid uid = MessageUid.of(Long.parseLong(doc.get(UID_FIELD)));
            MailboxId mailboxId = mailboxIdFactory.fromString(doc.get(MAILBOX_ID_FIELD));
            Optional<MessageId> messageId = toMessageId(Optional.ofNullable(doc.get(MESSAGE_ID_FIELD)));
            results.add(new SearchResult(messageId, mailboxId, uid));
        }
    } catch (IOException e) {
        throw new MailboxException("Unable to search the mailbox", e);
    }
    return results.build();
}
 
Example 14
Source Project: RDFS   File: TestDistributionPolicy.java    License: Apache License 2.0 5 votes vote down vote up
private void verify(Shard[] shards) throws IOException {
  // verify the index
  IndexReader[] readers = new IndexReader[shards.length];
  for (int i = 0; i < shards.length; i++) {
    Directory dir =
        new FileSystemDirectory(fs, new Path(shards[i].getDirectory()),
            false, conf);
    readers[i] = IndexReader.open(dir);
  }

  IndexReader reader = new MultiReader(readers);
  IndexSearcher searcher = new IndexSearcher(reader);
  Hits hits = searcher.search(new TermQuery(new Term("content", "apache")));
  assertEquals(0, hits.length());

  hits = searcher.search(new TermQuery(new Term("content", "hadoop")));
  assertEquals(numDocsPerRun / 2, hits.length());

  int[] counts = new int[numDocsPerRun];
  for (int i = 0; i < hits.length(); i++) {
    Document doc = hits.doc(i);
    counts[Integer.parseInt(doc.get("id"))]++;
  }

  for (int i = 0; i < numDocsPerRun; i++) {
    if (i % 2 == 0) {
      assertEquals(0, counts[i]);
    } else {
      assertEquals(1, counts[i]);
    }
  }

  searcher.close();
  reader.close();
}
 
Example 15
Source Project: imhotep   File: Benchmark.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws IOException, InterruptedException {
    final String luceneDir = args[0];
    final String simpleDir = args[1];

    final FlamdexReader reader1 = new LuceneFlamdexReader(IndexReader.open(luceneDir));
    final FlamdexReader reader2 = SimpleFlamdexReader.open(simpleDir);

    benchmark(reader1, reader2);
}
 
Example 16
private IndexReader createIndexReader(IndexType indexType) throws IOException {
    File dir = getIndexDirectory(indexType);
    return IndexReader.open(FSDirectory.open(dir), true);
}
 
Example 17
/**
 * Return a {@link Query} which is build based on the given {@link SearchQuery.FlagCriterion}. This is kind of a hack
 * as it will do a search for the flags in this method and
 */
private Query createFlagQuery(String flag, boolean isSet, Query inMailboxes, Collection<MessageUid> recentUids) throws MailboxException {
    BooleanQuery query = new BooleanQuery();
    
    if (isSet) {   
        query.add(new TermQuery(new Term(FLAGS_FIELD, flag)), BooleanClause.Occur.MUST);
    } else {
        // lucene does not support simple NOT queries so we do some nasty hack here
        BooleanQuery bQuery = new BooleanQuery();
        bQuery.add(new PrefixQuery(new Term(FLAGS_FIELD, "")), BooleanClause.Occur.MUST);
        bQuery.add(new TermQuery(new Term(FLAGS_FIELD, flag)),BooleanClause.Occur.MUST_NOT);
        
        query.add(bQuery, BooleanClause.Occur.MUST);
    }
    query.add(inMailboxes, BooleanClause.Occur.MUST);


    try (IndexSearcher searcher = new IndexSearcher(IndexReader.open(writer, true))) {
        Set<MessageUid> uids = new HashSet<>();

        // query for all the documents sorted by uid
        TopDocs docs = searcher.search(query, null, maxQueryResults, new Sort(UID_SORT));
        ScoreDoc[] sDocs = docs.scoreDocs;
        for (ScoreDoc sDoc : sDocs) {
            MessageUid uid = MessageUid.of(Long.parseLong(searcher.doc(sDoc.doc).get(UID_FIELD)));
            uids.add(uid);
        }
        
        // add or remove recent uids
        if (flag.equalsIgnoreCase("\\RECENT")) {
            if (isSet) {
                uids.addAll(recentUids);
            } else {
                uids.removeAll(recentUids);
            }
        }
        
        List<MessageRange> ranges = MessageRange.toRanges(new ArrayList<>(uids));
        UidRange[] nRanges = new UidRange[ranges.size()];
        for (int i = 0; i < ranges.size(); i++) {
            MessageRange range = ranges.get(i);
            nRanges[i] = new UidRange(range.getUidFrom(), range.getUidTo());
        }
        return createUidQuery((UidCriterion) SearchQuery.uid(nRanges));
    } catch (IOException e) {
        throw new MailboxException("Unable to search mailbox " + inMailboxes, e);
    }
}
 
Example 18
Source Project: olat   File: SearchSpellChecker.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Creates a new spell-check index based on search-index
 */
public void createSpellIndex() {
    if (isSpellCheckEnabled) {
        IndexReader indexReader = null;
        try {
            log.info("Start generating Spell-Index...");
            long startSpellIndexTime = 0;
            if (log.isDebugEnabled()) {
                startSpellIndexTime = System.currentTimeMillis();
            }
            final Directory indexDir = FSDirectory.open(new File(indexPath));
            indexReader = IndexReader.open(indexDir);
            // 1. Create content spellIndex
            final File spellDictionaryFile = new File(spellDictionaryPath);
            final Directory contentSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + CONTENT_PATH));// true
            final SpellChecker contentSpellChecker = new SpellChecker(contentSpellIndexDirectory);
            final Dictionary contentDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.CONTENT_FIELD_NAME);
            contentSpellChecker.indexDictionary(contentDictionary);
            // 2. Create title spellIndex
            final Directory titleSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + TITLE_PATH));// true
            final SpellChecker titleSpellChecker = new SpellChecker(titleSpellIndexDirectory);
            final Dictionary titleDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.TITLE_FIELD_NAME);
            titleSpellChecker.indexDictionary(titleDictionary);
            // 3. Create description spellIndex
            final Directory descriptionSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + DESCRIPTION_PATH));// true
            final SpellChecker descriptionSpellChecker = new SpellChecker(descriptionSpellIndexDirectory);
            final Dictionary descriptionDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.DESCRIPTION_FIELD_NAME);
            descriptionSpellChecker.indexDictionary(descriptionDictionary);
            // 4. Create author spellIndex
            final Directory authorSpellIndexDirectory = FSDirectory.open(new File(spellDictionaryPath + AUTHOR_PATH));// true
            final SpellChecker authorSpellChecker = new SpellChecker(authorSpellIndexDirectory);
            final Dictionary authorDictionary = new LuceneDictionary(indexReader, AbstractOlatDocument.AUTHOR_FIELD_NAME);
            authorSpellChecker.indexDictionary(authorDictionary);

            // Merge all part spell indexes (content,title etc.) to one common spell index
            final Directory spellIndexDirectory = FSDirectory.open(spellDictionaryFile);// true
            final IndexWriter merger = new IndexWriter(spellIndexDirectory, new StandardAnalyzer(Version.LUCENE_CURRENT), true, IndexWriter.MaxFieldLength.UNLIMITED);
            final Directory[] directories = { contentSpellIndexDirectory, titleSpellIndexDirectory, descriptionSpellIndexDirectory, authorSpellIndexDirectory };
            merger.addIndexesNoOptimize(directories);
            merger.optimize();
            merger.close();
            spellChecker = new SpellChecker(spellIndexDirectory);
            spellChecker.setAccuracy(0.7f);
            if (log.isDebugEnabled()) {
                log.debug("SpellIndex created in " + (System.currentTimeMillis() - startSpellIndexTime) + "ms");
            }
            log.info("New generated Spell-Index ready to use.");
        } catch (final IOException ioEx) {
            log.warn("Can not create SpellIndex", ioEx);
        } finally {
            if (indexReader != null) {
                try {
                    indexReader.close();
                } catch (final IOException e) {
                    log.warn("Can not close indexReader properly", e);
                }
            }
        }
    }
}
 
Example 19
@Test
    public void testCloseSessionDuringFTGS() throws ImhotepOutOfMemoryException, IOException, InterruptedException {
        String tempDir = Files.getTempDirectory("asdf", "");
        try {
            IndexWriter w = new IndexWriter(tempDir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED);

            Random rand = new Random();
            for (int i = 0; i < 1000000; ++i) {
                int numTerms = rand.nextInt(5) + 5;
                Document doc = new Document();
                for (int t = 0; t < numTerms; ++t) {
                    doc.add(new Field("sf1", Integer.toString(rand.nextInt(10000)), Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS));
                }
                w.addDocument(doc);
            }

            w.close();

            final AtomicBoolean closed = new AtomicBoolean(false);
            FlamdexReader r = new LuceneFlamdexReader(IndexReader.open(tempDir)) {
                @Override
                public void close() throws IOException {
                    super.close();
                    closed.set(true);
                }
            };
            final ExecutorService executor = Executors.newCachedThreadPool();
            try {
                ImhotepSession session =
                        new MTImhotepMultiSession(new ImhotepLocalSession[] { new ImhotepLocalSession(r) },
                                                  new MemoryReservationContext(
                                                                               new ImhotepMemoryPool(
                                                                                                     Long.MAX_VALUE)),
                                                  executor, null);
//                FTGSIterator iter = session.getFTGSIterator(new String[]{}, new String[]{"sf1"}); //TODO fix this
                session.close();
                assertTrue(closed.get());
            } finally {
                executor.shutdown();
            }
        } finally {
            Files.delete(tempDir);
        }
    }
 
Example 20
/**
 * Creates the object and opens the index file.
 *
 * @param dictType
 *            the dictionary we will use for the search.
 * @param dictionaryPath
 *            overrides default dictionary location if non-null. An absolute
 *            os-specific path, e.g. /sdcard/aedict/index.
 * @param sort if true then the result list is always sorted.
 * @throws IOException
 *             on I/O error.
 */
public LuceneSearch(final DictTypeEnum dictType, final String dictionaryPath, final boolean sort) throws IOException {
    this.dictType = dictType;
    directory = FSDirectory.open(new File(dictionaryPath != null ? dictionaryPath : dictType.getDefaultDictionaryPath()));
    reader = IndexReader.open(directory, true);
    searcher = new IndexSearcher(reader);
    parser = new QueryParser(LUCENE_VERSION, "contents", new StandardAnalyzer(LUCENE_VERSION));
    this.sort = sort;
}