org.apache.lucene.store.FSDirectory Java Examples

The following examples show how to use org.apache.lucene.store.FSDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: localization_nifi   Author: wangrenlei   File: TestPersistentProvenanceRepository.java    License: Apache License 2.0 7 votes vote down vote up
private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
    try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory))) {
        final IndexSearcher searcher = new IndexSearcher(directoryReader);

        final Analyzer analyzer = new SimpleAnalyzer();
        final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);

        final Query q = new Query("");
        q.setMaxResults(1000);
        final TopDocs topDocs = searcher.search(luceneQuery, 1000);

        final List<Document> docs = new ArrayList<>();
        for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
            final int docId = scoreDoc.doc;
            final Document d = directoryReader.document(docId);
            docs.add(d);
        }

        return docs;
    }
}
 
Example #2
Source Project: subsonic   Author: sindremehus   File: SearchService.java    License: GNU General Public License v3.0 6 votes vote down vote up
private void removeLocks() {
    for (IndexType indexType : IndexType.values()) {
        Directory dir = null;
        try {
            dir = FSDirectory.open(getIndexDirectory(indexType));
            if (IndexWriter.isLocked(dir)) {
                IndexWriter.unlock(dir);
                LOG.info("Removed Lucene lock file in " + dir);
            }
        } catch (Exception x) {
            LOG.warn("Failed to remove Lucene lock file in " + dir, x);
        } finally {
            FileUtil.closeQuietly(dir);
        }
    }
}
 
Example #3
Source Project: lucene-solr   Author: apache   File: TestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static boolean hasWindowsFS(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    Path path = ((FSDirectory) dir).getDirectory();
    FileSystem fs = path.getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof WindowsFS) {
        return true;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example #4
Source Project: lucene4ir   Author: lucene4ir   File: DocumentIndexer.java    License: Apache License 2.0 6 votes vote down vote up
public void createWriter(String indexPath){
    /*
    The indexPath specifies where to create the index
     */

    // I am can imagine that there are lots of ways to create indexers -
    // We could add in some parameters to customize its creation

    try {
        Directory dir = FSDirectory.open(Paths.get(indexPath));
        System.out.println("Indexing to directory '" + indexPath + "'...");

        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
        writer = new IndexWriter(dir, iwc);

    } catch (IOException e){
        e.printStackTrace();
        System.exit(1);
    }
}
 
Example #5
Source Project: lucene-solr   Author: apache   File: GetTermInfo.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  
  FSDirectory dir = null;
  String inputStr = null;
  String field = null;
  
  if (args.length == 3) {
    dir = FSDirectory.open(Paths.get(args[0]));
    field = args[1];
    inputStr = args[2];
  } else {
    usage();
    System.exit(1);
  }
    
  getTermInfo(dir,new Term(field, inputStr));
}
 
Example #6
Source Project: lucene-solr   Author: apache   File: TestStressThreadBackup.java    License: Apache License 2.0 6 votes vote down vote up
/** 
 * Validates a backup dir exists, passes check index, and contains a number of "real" documents
 * that match it's name
 * 
 * @see #getNumRealDocsFromBackupName
 */
private void validateBackup(final File backup) throws IOException {
  log.info("Checking Validity of {}", backup);
  assertTrue(backup.toString() + ": isDir?", backup.isDirectory());
  final Matcher m = ENDS_WITH_INT_DIGITS.matcher(backup.getName());
  assertTrue("Backup dir name does not end with int digits: " + backup.toString(), m.find());
  final int numRealDocsExpected = Integer.parseInt(m.group());
  
  try (Directory dir = FSDirectory.open(backup.toPath())) {
    TestUtil.checkIndex(dir, true, true, null);
    try (DirectoryReader r = DirectoryReader.open(dir)) {
      assertEquals("num real docs in " + backup.toString(),
                   numRealDocsExpected, r.docFreq(new Term("type_s","real")));
    }
  }
}
 
Example #7
Source Project: lucene-solr   Author: apache   File: PerfRunData.java    License: Apache License 2.0 6 votes vote down vote up
private Directory createDirectory(boolean eraseIndex, String dirName,
    String dirParam) throws IOException {
  String dirImpl = config.get(dirParam, DEFAULT_DIRECTORY);
  if ("FSDirectory".equals(dirImpl)) {
    Path workDir = Paths.get(config.get("work.dir", "work"));
    Path indexDir = workDir.resolve(dirName);
    if (eraseIndex && Files.exists(indexDir)) {
      IOUtils.rm(indexDir);
    }
    Files.createDirectories(indexDir);
    return FSDirectory.open(indexDir);
  }

  if ("RAMDirectory".equals(dirImpl)) {
    throw new IOException("RAMDirectory has been removed, use ByteBuffersDirectory.");
  }

  if ("ByteBuffersDirectory".equals(dirImpl)) {
    return new ByteBuffersDirectory();
  }

  throw new IOException("Directory type not supported: " + dirImpl);
}
 
Example #8
Source Project: SciGraph   Author: SciGraph   File: VocabularyNeo4jImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public VocabularyNeo4jImpl(GraphDatabaseService graph,
    @Nullable @IndicatesNeo4jGraphLocation String neo4jLocation, CurieUtil curieUtil,
    NodeTransformer transformer) throws IOException {
  this.graph = graph;
  this.curieUtil = curieUtil;
  this.transformer = transformer;
  if (null != neo4jLocation) {
    Directory indexDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/node/node_auto_index"))
            .toPath());
    Directory spellDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/spellchecker"))
            .toPath());
    spellChecker = new SpellChecker(spellDirectory);
    try (IndexReader reader = DirectoryReader.open(indexDirectory)) {
      IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer());
      spellChecker.indexDictionary(new LuceneDictionary(reader, NodeProperties.LABEL
          + LuceneUtils.EXACT_SUFFIX), config, true);
    }
  } else {
    spellChecker = null;
  }
}
 
Example #9
Source Project: mysiteforme   Author: wangl1989   File: LuceneSearch.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 根据ID更新搜索内容
 * @param blogArticle
 * @throws IOException
 */
public static void updateIndexById(BlogArticle blogArticle) throws IOException{
    Directory directory = FSDirectory.open(Paths.get(dir));// 打开文件索引目录
    Analyzer analyzer = new IKAnalyzer();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
    //创建索引写入对象
    IndexWriter writer = new IndexWriter(directory,indexWriterConfig);
    Document doc = new Document();
    doc.add(new LongPoint("id",blogArticle.getId()));
    doc.add(new TextField("title",blogArticle.getTitle(), Field.Store.YES));
    doc.add(new TextField("marks",blogArticle.getMarks()==null?"":blogArticle.getMarks(),Field.Store.YES));
    doc.add(new TextField("text",blogArticle.getText()==null?"":blogArticle.getText(),Field.Store.YES));
    doc.add(new StoredField("href",blogArticle.getBlogChannel().getHref()));
    doc.add(new StoredField("show_pic",blogArticle.getShowPic()==null?"":blogArticle.getShowPic()));
    writer.updateDocument(new Term("id", blogArticle.getId().toString()), doc);
    writer.commit();// 提交
    writer.close();// 关闭
}
 
Example #10
Source Project: RedisDirectory   Author: shijiebei2009   File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testMMapDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    FSDirectory open = FSDirectory.open(Paths.get("E:/testlucene"));
    IndexWriter indexWriter = new IndexWriter(open, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("MMapDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(open));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("MMapDirectory search consumes {}ms!", (end - start));
}
 
Example #11
Source Project: restcommander   Author: eBay   File: FilesystemStore.java    License: Apache License 2.0 6 votes vote down vote up
private IndexWriter getIndexWriter(String name) {
    try {
        if (!indexWriters.containsKey(name)) {
            synchronized (this) {
                File root = new File(DATA_PATH, name);
                if (!root.exists())
                    root.mkdirs();
                if (new File(root, "write.lock").exists())
                    new File(root, "write.lock").delete();
                IndexWriter writer = new IndexWriter(FSDirectory.open(root), Search.getAnalyser(), MaxFieldLength.UNLIMITED);
                indexWriters.put(name, writer);
            }
        }
        return indexWriters.get(name);
    } catch (Exception e) {
        throw new UnexpectedException(e);
    }
}
 
Example #12
Source Project: Java-Data-Science-Cookbook   Author: PacktPublishing   File: SearchFiles.java    License: MIT License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(INDEX_DIRECTORY)));
	IndexSearcher indexSearcher = new IndexSearcher(reader);

	Analyzer analyzer = new StandardAnalyzer();
	QueryParser queryParser = new QueryParser(FIELD_CONTENTS, analyzer);
	String searchString = "shakespeare";
	Query query = queryParser.parse(searchString);

	TopDocs results = indexSearcher.search(query, 5);
	ScoreDoc[] hits = results.scoreDocs;

	int numTotalHits = results.totalHits;
	System.out.println(numTotalHits + " total matching documents");

	for(int i=0;i<hits.length;++i) {
		int docId = hits[i].doc;
		Document d = indexSearcher.doc(docId);
		System.out.println((i + 1) + ". " + d.get("path") + " score=" + hits[i].score);
	}
}
 
Example #13
Source Project: lucene-solr   Author: apache   File: SolrDeletionPolicy.java    License: Apache License 2.0 6 votes vote down vote up
private String getId(IndexCommit commit) {
  StringBuilder sb = new StringBuilder();
  Directory dir = commit.getDirectory();

  // For anything persistent, make something that will
  // be the same, regardless of the Directory instance.
  if (dir instanceof FSDirectory) {
    FSDirectory fsd = (FSDirectory) dir;
    File fdir = fsd.getDirectory().toFile();
    sb.append(fdir.getPath());
  } else {
    sb.append(dir);
  }

  sb.append('/');
  sb.append(commit.getGeneration());
  return sb.toString();
}
 
Example #14
Source Project: dexter   Author: dexter   File: LuceneHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Opens or creates a lucene index in the given directory
 * 
 * @param wikiIdtToLuceneIdSerialization
 *            - the file containing the serialized mapping between wiki-id
 *            and Lucene documents ids
 * 
 * @param indexPath
 *            - the path of the directory with the Lucene's index
 */
protected LuceneHelper(File wikiIdtToLuceneIdSerialization, File indexPath) {
	logger.info("opening lucene index in folder {}", indexPath);
	config = new IndexWriterConfig(Version.LUCENE_41, ANALYZER);
	this.wikiIdtToLuceneIdSerialization = wikiIdtToLuceneIdSerialization;

	BooleanQuery.setMaxClauseCount(1000);

	try {
		index = FSDirectory.open(indexPath);
		// writer.commit();
	} catch (Exception e) {
		logger.error("opening the index: {}", e.toString());
		System.exit(1);
	}

	summarizer = new ArticleSummarizer();
	writer = getWriter();
	collectionSize = writer.numDocs();
	wikiIdToLuceneId = Collections.emptyMap();
}
 
Example #15
Source Project: bluima   Author: BlueBrain   File: Txt2PubmedIdIndexer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(UimaContext context)
        throws ResourceInitializationException {
    super.initialize(context);
    try {
        // create writer
        Directory dir;
        dir = FSDirectory.open(new File(INDEX_PATH));
        Analyzer analyzer = getAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(
                Version.LUCENE_41, analyzer);
        iwc.setOpenMode(OpenMode.CREATE);
        indexWriter = new IndexWriter(dir, iwc);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example #16
Source Project: tagme   Author: gammaliu   File: AnchorIndexer.java    License: Apache License 2.0 6 votes vote down vote up
private IndexSearcher openWikipediaIndex(String lang) throws IOException{
	
	
	File indexDir = RepositoryDirs.WIKIPEDIA.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	if (indexSize < maxMemory * GAP_FACTOR){
		
		log.info("MaxMemory is enough, loading Wikipedia index...");
		IndexReader r = IndexReader.open(new RAMDirectory(FSDirectory.open(indexDir)), true);
		log.info("WikipediaIndex loaded.");
		return new IndexSearcher(r);
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load WikipediaIndex (about "+indexSize/1000000+"Mb)");
		return Indexes.getSearcher(RepositoryDirs.WIKIPEDIA.getPath(lang));
	}
}
 
Example #17
Source Project: incubator-retired-blur   Author: apache   File: BlockDirectory.java    License: Apache License 2.0 6 votes vote down vote up
private long getFileModified(String name) throws IOException {
  if (_directory instanceof FSDirectory) {
    File directory = ((FSDirectory) _directory).getDirectory();
    File file = new File(directory, name);
    if (!file.exists()) {
      throw new FileNotFoundException("File [" + name + "] not found");
    }
    return file.lastModified();
  } else if (_directory instanceof HdfsDirectory) {
    return ((HdfsDirectory) _directory).getFileModified(name);
  } else if (_directory instanceof LastModified) {
    return ((LastModified) _directory).getFileModified(name);
  } else {
    throw new RuntimeException("Not supported");
  }
}
 
Example #18
Source Project: webdsl   Author: webdsl   File: AbstractIndexManager.java    License: Apache License 2.0 6 votes vote down vote up
protected static boolean clearIndex(File path) {
	try {
		if (path == null || !path.exists())
			return true; // if path doesnt exist, then there is nothing to
							// clear

		FSDirectory indexDir = new FSDirectoryProvider().getDirectory();
		IndexWriter writer = new IndexWriter(indexDir.open(path),
				new IndexWriterConfig(Version.LUCENE_CURRENT,
						new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
		writer.deleteAll();
		writer.close();
		return true;
	} catch (Exception ex) {
		org.webdsl.logging.Logger.error(
				"Error while clearing index on location: " + path, ex);
		return false;
	}

}
 
Example #19
Source Project: lucene-solr   Author: apache   File: TestUtil.java    License: Apache License 2.0 6 votes vote down vote up
/** Returns true if VirusCheckingFS is in use and was in fact already enabled */
public static boolean disableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        boolean isEnabled = vfs.isEnabled();
        vfs.disable();
        return isEnabled;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example #20
Source Project: tephra   Author: heisedebaise   File: LuceneHelperImpl.java    License: MIT License 6 votes vote down vote up
private synchronized Directory get(String key) {
    if (map == null) {
        map = new ConcurrentHashMap<>();
        BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
    }

    return map.computeIfAbsent(key, k -> {
        Path path = Paths.get(context.getAbsoluteRoot(), root, k, "index");
        io.mkdirs(path.toFile());
        if (logger.isInfoEnable())
            logger.info("设置Lucene索引根目录[{}:{}]。", k, path);
        try {
            return FSDirectory.open(path);
        } catch (IOException e) {
            logger.warn(e, "打开Lucene索引目录[{}:{}]时发生异常!", k, path);

            return null;
        }
    });
}
 
Example #21
Source Project: semanticvectors   Author: semanticvectors   File: VectorStoreReaderLucene.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public VectorStoreReaderLucene(String vectorFileName, FlagConfig flagConfig) throws IOException {
  this.flagConfig = flagConfig;
  this.vectorFileName = vectorFileName;
  this.vectorFile = new File(vectorFileName);
  try {
    String parentPath = this.vectorFile.getParent();
    if (parentPath == null) parentPath = "";
    this.directory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));  // Old from FSDirectory impl.
    // Read number of dimension from header information.
    this.threadLocalIndexInput = new ThreadLocal<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        try {
          return directory.openInput(vectorFile.getName(), IOContext.READ);
        } catch (IOException e) {
          throw new RuntimeException(e.getMessage(), e);
        }
      }
    };
    readHeadersFromIndexInput(flagConfig);
  } catch (IOException e) {
    logger.warning("Cannot open file: " + this.vectorFileName + "\n" + e.getMessage());
    throw e;
  }
}
 
Example #22
Source Project: bioasq   Author: oaqa   File: LuceneDocumentRetrievalExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 100);
  // query constructor
  constructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          BooleanBagOfPhraseQueryStringConstructor.class, QueryStringConstructor.class);
  // lucene
  Analyzer analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  String[] fields = UimaContextHelper.getConfigParameterStringArrayValue(context, "fields");
  parser = new MultiFieldQueryParser(fields, analyzer);
  String index = UimaContextHelper.getConfigParameterStringValue(context, "index");
  try {
    reader = DirectoryReader.open(FSDirectory.open(Paths.get(index)));
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
  searcher = new IndexSearcher(reader);
  idFieldName = UimaContextHelper.getConfigParameterStringValue(context, "id-field", null);
  titleFieldName = UimaContextHelper.getConfigParameterStringValue(context, "title-field", null);
  textFieldName = UimaContextHelper.getConfigParameterStringValue(context, "text-field", null);
  uriPrefix = UimaContextHelper.getConfigParameterStringValue(context, "uri-prefix", null);
}
 
Example #23
Source Project: RedisDirectory   Author: shijiebei2009   File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testMMapDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    FSDirectory open = FSDirectory.open(Paths.get("E:/testlucene"));
    IndexWriter indexWriter = new IndexWriter(open, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("MMapDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(open));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("MMapDirectory search consumes {}ms!", (end - start));
}
 
Example #24
Source Project: ApiManager   Author: EhsanTang   File: LuceneSearchService.java    License: GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public boolean delete(SearchDto searchDto) throws IOException {
	IndexWriter writer = null;
	try {
		IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
		conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
		writer = new IndexWriter(FSDirectory.open(Paths.get(settingCache.get(ISetting.S_LUCENE_DIR).getValue())), conf);
		writer.deleteDocuments(new Term(ID, searchDto.getId()));
	} catch (Exception e) {
		e.printStackTrace();
		stringCache.add(IConst.C_CACHE_ERROR_TIP, "Lucene删除异常,请联系管理员查看日志,错误信息:" + e.getMessage());
	} finally {
		if (writer != null) {
			writer.close();
		}
	}
	return true;
}
 
Example #25
Source Project: SourcererCC   Author: Mondego   File: CodeSearcher.java    License: GNU General Public License v3.0 6 votes vote down vote up
public CodeSearcher(String indexDir, String field) {
    logger.info("index directory: "+ indexDir);
    this.field = field;
    this.indexDir = indexDir;
    try {
        this.reader = DirectoryReader.open(FSDirectory.open(new File(
                this.indexDir)));
    } catch (IOException e) {
        logger.error("cant get the reader to index dir, exiting, "
                + indexDir);
        e.printStackTrace();
        System.exit(1);
    }
    this.searcher = new IndexSearcher(this.reader);
    this.analyzer = new KeywordAnalyzer();//
            //new WhitespaceAnalyzer(Version.LUCENE_46); // TODO: pass
                                                               // the
                                                               // analyzer
                                                               // as
                                                               // argument
                                                               // to
                                                               // constructor
    new CloneHelper(); // i don't remember why we are making this object?
    this.queryParser = new QueryParser(Version.LUCENE_46, this.field,
            analyzer);
}
 
Example #26
Source Project: ES-Fastloader   Author: didi   File: AppendLuceneTransportAction.java    License: Apache License 2.0 5 votes vote down vote up
private void doExecuteCore(AppendLuceneRequest request, ActionListener<AppendLuceneResponse> listener) {
    try {
        // 对请求做check
        request.check();

        // 获得shard信息
        ShardId shardId = new ShardId(request.indexName, request.uuid, request.shardId);
        IndexShard shard = indicesService.getShardOrNull(shardId);
        if(shard==null) {
            throw new Exception("shard not found, indexName:" + request.indexName + ", shardId:" + request.shardId);
        }

        // 获得lucene的IndexWriter对象
        /* FIXME 这里需要修改es的代码, 将lucene的IndexWriter对象暴露给plugin使用  */
        InternalEngine engine = (InternalEngine) shard.getEngineOrNull();
        IndexWriter indexWriter = engine.getIndexWriter();

        // 处理主键冲突情况
        long deleteCount = -1;
        List<String> appendDirs = request.getAppendDirs();
        if(request.primeKey!=null && request.primeKey.length()>0) {
            deleteCount = doPrimerKey(appendDirs, indexWriter, request.primeKey);
        }

        // 将新的lucene文件加入到shard中
        Directory[] indexes = new Directory[appendDirs.size()];
        for(int i=0; i<appendDirs.size(); i++) {
            indexes[i] = FSDirectory.open(Paths.get(appendDirs.get(i)));
        }
        indexWriter.addIndexes(indexes);
        indexWriter.commit();

        // 构建response
        AppendLuceneResponse response = new AppendLuceneResponse();
        response.deleteCount = deleteCount;
        listener.onResponse(response);
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
 
Example #27
Source Project: lucene-geo-gazetteer   Author: chrismattmann   File: GeoNameResolver.java    License: Apache License 2.0 5 votes vote down vote up
private IndexReader createIndexReader(String indexerPath) throws IOException {
	File indexfile = new File(indexerPath);
	indexDir = FSDirectory.open(indexfile.toPath());


	if (!DirectoryReader.indexExists(indexDir)) {
		LOG.log(Level.SEVERE,
				"No Lucene Index Dierctory Found, Invoke indexBuild() First !");
		System.exit(1);
	}

	return DirectoryReader.open(indexDir);
}
 
Example #28
Source Project: nifi   Author: apache   File: StandardIndexManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public EventIndexSearcher borrowIndexSearcher(final File indexDir) throws IOException {
    final File absoluteFile = indexDir.getAbsoluteFile();

    final IndexWriterCount writerCount;
    synchronized (writerCounts) {
        writerCount = writerCounts.remove(absoluteFile);

        if (writerCount != null) {
            // Increment writer count and create an Index Searcher based on the writer
            writerCounts.put(absoluteFile, new IndexWriterCount(writerCount.getWriter(), writerCount.getAnalyzer(),
                writerCount.getDirectory(), writerCount.getCount() + 1, writerCount.isCloseableWhenUnused()));
        }
    }

    final DirectoryReader directoryReader;
    if (writerCount == null) {
        logger.trace("Creating index searcher for {}", indexDir);
        final Directory directory = FSDirectory.open(indexDir.toPath());
        directoryReader = DirectoryReader.open(directory);
    } else {
        final EventIndexWriter eventIndexWriter = writerCount.getWriter();
        directoryReader = DirectoryReader.open(eventIndexWriter.getIndexWriter(), false, false);
    }

    final IndexSearcher searcher = new IndexSearcher(directoryReader, this.searchExecutor);

    logger.trace("Created index searcher {} for {}", searcher, indexDir);
    return new LuceneEventIndexSearcher(searcher, indexDir, null, directoryReader);
}
 
Example #29
Source Project: taoshop   Author: u014427391   File: SearchBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public static void doSearch(String indexDir , String queryStr) throws IOException, ParseException, InvalidTokenOffsetsException {
    Directory directory = FSDirectory.open(Paths.get(indexDir));
    DirectoryReader reader = DirectoryReader.open(directory);
    IndexSearcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new SmartChineseAnalyzer();
    QueryParser parser = new QueryParser("tcontent",analyzer);
    Query query = parser.parse(queryStr);

    long startTime = System.currentTimeMillis();
    TopDocs docs = searcher.search(query,10);

    System.out.println("查找"+queryStr+"所用时间:"+(System.currentTimeMillis()-startTime));
    System.out.println("查询到"+docs.totalHits+"条记录");

    //加入高亮显示的
    SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<b><font color=red>","</font></b>");
    QueryScorer scorer = new QueryScorer(query);//计算查询结果最高的得分
    Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);//根据得分算出一个片段
    Highlighter highlighter = new Highlighter(simpleHTMLFormatter,scorer);
    highlighter.setTextFragmenter(fragmenter);//设置显示高亮的片段

    //遍历查询结果
    for(ScoreDoc scoreDoc : docs.scoreDocs){
        Document doc = searcher.doc(scoreDoc.doc);
        System.out.println(doc.get("title"));
        System.out.println(doc.get("tcontent"));
        String tcontent = doc.get("tcontent");
        if(tcontent != null){
            TokenStream tokenStream =  analyzer.tokenStream("tcontent", new StringReader(tcontent));
            String summary = highlighter.getBestFragment(tokenStream, tcontent);
            System.out.println(summary);
        }
    }
    reader.close();
}
 
Example #30
Source Project: Siamese   Author: UCL-CREST   File: Siamese.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * Read the ES index file
 * @param indexName the name of the index
 */
private void readESIndex(String indexName) {
    String indexFile = elasticsearchLoc + "/data/stackoverflow/nodes/0/indices/"
            + indexName + "/0/index";
    try {
        esIndexRader = DirectoryReader.open(FSDirectory.open(Paths.get(indexFile)));
    } catch (IOException e) {
        e.printStackTrace();
    }
}