Java Code Examples for org.apache.lucene.store.FSDirectory

The following examples show how to use org.apache.lucene.store.FSDirectory. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: subsonic   Source File: SearchService.java    License: GNU General Public License v3.0 6 votes vote down vote up
private void removeLocks() {
    for (IndexType indexType : IndexType.values()) {
        Directory dir = null;
        try {
            dir = FSDirectory.open(getIndexDirectory(indexType));
            if (IndexWriter.isLocked(dir)) {
                IndexWriter.unlock(dir);
                LOG.info("Removed Lucene lock file in " + dir);
            }
        } catch (Exception x) {
            LOG.warn("Failed to remove Lucene lock file in " + dir, x);
        } finally {
            FileUtil.closeQuietly(dir);
        }
    }
}
 
Example 2
Source Project: lucene-solr   Source File: TestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static boolean hasWindowsFS(Directory dir) {
  dir = FilterDirectory.unwrap(dir);
  if (dir instanceof FSDirectory) {
    Path path = ((FSDirectory) dir).getDirectory();
    FileSystem fs = path.getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof WindowsFS) {
        return true;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example 3
Source Project: lucene4ir   Source File: DocumentIndexer.java    License: Apache License 2.0 6 votes vote down vote up
public void createWriter(String indexPath){
    /*
    The indexPath specifies where to create the index
     */

    // I am can imagine that there are lots of ways to create indexers -
    // We could add in some parameters to customize its creation

    try {
        Directory dir = FSDirectory.open(Paths.get(indexPath));
        System.out.println("Indexing to directory '" + indexPath + "'...");

        IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
        iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
        writer = new IndexWriter(dir, iwc);

    } catch (IOException e){
        e.printStackTrace();
        System.exit(1);
    }
}
 
Example 4
Source Project: lucene-solr   Source File: GetTermInfo.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
  
  FSDirectory dir = null;
  String inputStr = null;
  String field = null;
  
  if (args.length == 3) {
    dir = FSDirectory.open(Paths.get(args[0]));
    field = args[1];
    inputStr = args[2];
  } else {
    usage();
    System.exit(1);
  }
    
  getTermInfo(dir,new Term(field, inputStr));
}
 
Example 5
Source Project: lucene-solr   Source File: TestStressThreadBackup.java    License: Apache License 2.0 6 votes vote down vote up
/** 
 * Validates a backup dir exists, passes check index, and contains a number of "real" documents
 * that match it's name
 * 
 * @see #getNumRealDocsFromBackupName
 */
private void validateBackup(final File backup) throws IOException {
  log.info("Checking Validity of {}", backup);
  assertTrue(backup.toString() + ": isDir?", backup.isDirectory());
  final Matcher m = ENDS_WITH_INT_DIGITS.matcher(backup.getName());
  assertTrue("Backup dir name does not end with int digits: " + backup.toString(), m.find());
  final int numRealDocsExpected = Integer.parseInt(m.group());
  
  try (Directory dir = FSDirectory.open(backup.toPath())) {
    TestUtil.checkIndex(dir, true, true, null);
    try (DirectoryReader r = DirectoryReader.open(dir)) {
      assertEquals("num real docs in " + backup.toString(),
                   numRealDocsExpected, r.docFreq(new Term("type_s","real")));
    }
  }
}
 
Example 6
private List<Document> runQuery(final File indexDirectory, final List<File> storageDirs, final String query) throws IOException, ParseException {
    try (final DirectoryReader directoryReader = DirectoryReader.open(FSDirectory.open(indexDirectory))) {
        final IndexSearcher searcher = new IndexSearcher(directoryReader);

        final Analyzer analyzer = new SimpleAnalyzer();
        final org.apache.lucene.search.Query luceneQuery = new QueryParser("uuid", analyzer).parse(query);

        final Query q = new Query("");
        q.setMaxResults(1000);
        final TopDocs topDocs = searcher.search(luceneQuery, 1000);

        final List<Document> docs = new ArrayList<>();
        for (final ScoreDoc scoreDoc : topDocs.scoreDocs) {
            final int docId = scoreDoc.doc;
            final Document d = directoryReader.document(docId);
            docs.add(d);
        }

        return docs;
    }
}
 
Example 7
Source Project: lucene-solr   Source File: PerfRunData.java    License: Apache License 2.0 6 votes vote down vote up
private Directory createDirectory(boolean eraseIndex, String dirName,
    String dirParam) throws IOException {
  String dirImpl = config.get(dirParam, DEFAULT_DIRECTORY);
  if ("FSDirectory".equals(dirImpl)) {
    Path workDir = Paths.get(config.get("work.dir", "work"));
    Path indexDir = workDir.resolve(dirName);
    if (eraseIndex && Files.exists(indexDir)) {
      IOUtils.rm(indexDir);
    }
    Files.createDirectories(indexDir);
    return FSDirectory.open(indexDir);
  }

  if ("RAMDirectory".equals(dirImpl)) {
    throw new IOException("RAMDirectory has been removed, use ByteBuffersDirectory.");
  }

  if ("ByteBuffersDirectory".equals(dirImpl)) {
    return new ByteBuffersDirectory();
  }

  throw new IOException("Directory type not supported: " + dirImpl);
}
 
Example 8
Source Project: SciGraph   Source File: VocabularyNeo4jImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public VocabularyNeo4jImpl(GraphDatabaseService graph,
    @Nullable @IndicatesNeo4jGraphLocation String neo4jLocation, CurieUtil curieUtil,
    NodeTransformer transformer) throws IOException {
  this.graph = graph;
  this.curieUtil = curieUtil;
  this.transformer = transformer;
  if (null != neo4jLocation) {
    Directory indexDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/node/node_auto_index"))
            .toPath());
    Directory spellDirectory =
        FSDirectory.open((new File(new File(neo4jLocation), "index/lucene/spellchecker"))
            .toPath());
    spellChecker = new SpellChecker(spellDirectory);
    try (IndexReader reader = DirectoryReader.open(indexDirectory)) {
      IndexWriterConfig config = new IndexWriterConfig(new KeywordAnalyzer());
      spellChecker.indexDictionary(new LuceneDictionary(reader, NodeProperties.LABEL
          + LuceneUtils.EXACT_SUFFIX), config, true);
    }
  } else {
    spellChecker = null;
  }
}
 
Example 9
Source Project: mysiteforme   Source File: LuceneSearch.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 根据ID更新搜索内容
 * @param blogArticle
 * @throws IOException
 */
public static void updateIndexById(BlogArticle blogArticle) throws IOException{
    Directory directory = FSDirectory.open(Paths.get(dir));// 打开文件索引目录
    Analyzer analyzer = new IKAnalyzer();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(analyzer);
    //创建索引写入对象
    IndexWriter writer = new IndexWriter(directory,indexWriterConfig);
    Document doc = new Document();
    doc.add(new LongPoint("id",blogArticle.getId()));
    doc.add(new TextField("title",blogArticle.getTitle(), Field.Store.YES));
    doc.add(new TextField("marks",blogArticle.getMarks()==null?"":blogArticle.getMarks(),Field.Store.YES));
    doc.add(new TextField("text",blogArticle.getText()==null?"":blogArticle.getText(),Field.Store.YES));
    doc.add(new StoredField("href",blogArticle.getBlogChannel().getHref()));
    doc.add(new StoredField("show_pic",blogArticle.getShowPic()==null?"":blogArticle.getShowPic()));
    writer.updateDocument(new Term("id", blogArticle.getId().toString()), doc);
    writer.commit();// 提交
    writer.close();// 关闭
}
 
Example 10
Source Project: RedisDirectory   Source File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testMMapDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    FSDirectory open = FSDirectory.open(Paths.get("E:/testlucene"));
    IndexWriter indexWriter = new IndexWriter(open, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("MMapDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(open));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("MMapDirectory search consumes {}ms!", (end - start));
}
 
Example 11
Source Project: restcommander   Source File: FilesystemStore.java    License: Apache License 2.0 6 votes vote down vote up
private IndexWriter getIndexWriter(String name) {
    try {
        if (!indexWriters.containsKey(name)) {
            synchronized (this) {
                File root = new File(DATA_PATH, name);
                if (!root.exists())
                    root.mkdirs();
                if (new File(root, "write.lock").exists())
                    new File(root, "write.lock").delete();
                IndexWriter writer = new IndexWriter(FSDirectory.open(root), Search.getAnalyser(), MaxFieldLength.UNLIMITED);
                indexWriters.put(name, writer);
            }
        }
        return indexWriters.get(name);
    } catch (Exception e) {
        throw new UnexpectedException(e);
    }
}
 
Example 12
Source Project: Java-Data-Science-Cookbook   Source File: SearchFiles.java    License: MIT License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(INDEX_DIRECTORY)));
	IndexSearcher indexSearcher = new IndexSearcher(reader);

	Analyzer analyzer = new StandardAnalyzer();
	QueryParser queryParser = new QueryParser(FIELD_CONTENTS, analyzer);
	String searchString = "shakespeare";
	Query query = queryParser.parse(searchString);

	TopDocs results = indexSearcher.search(query, 5);
	ScoreDoc[] hits = results.scoreDocs;

	int numTotalHits = results.totalHits;
	System.out.println(numTotalHits + " total matching documents");

	for(int i=0;i<hits.length;++i) {
		int docId = hits[i].doc;
		Document d = indexSearcher.doc(docId);
		System.out.println((i + 1) + ". " + d.get("path") + " score=" + hits[i].score);
	}
}
 
Example 13
Source Project: lucene-solr   Source File: SolrDeletionPolicy.java    License: Apache License 2.0 6 votes vote down vote up
private String getId(IndexCommit commit) {
  StringBuilder sb = new StringBuilder();
  Directory dir = commit.getDirectory();

  // For anything persistent, make something that will
  // be the same, regardless of the Directory instance.
  if (dir instanceof FSDirectory) {
    FSDirectory fsd = (FSDirectory) dir;
    File fdir = fsd.getDirectory().toFile();
    sb.append(fdir.getPath());
  } else {
    sb.append(dir);
  }

  sb.append('/');
  sb.append(commit.getGeneration());
  return sb.toString();
}
 
Example 14
Source Project: dexter   Source File: LuceneHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Opens or creates a lucene index in the given directory
 * 
 * @param wikiIdtToLuceneIdSerialization
 *            - the file containing the serialized mapping between wiki-id
 *            and Lucene documents ids
 * 
 * @param indexPath
 *            - the path of the directory with the Lucene's index
 */
protected LuceneHelper(File wikiIdtToLuceneIdSerialization, File indexPath) {
	logger.info("opening lucene index in folder {}", indexPath);
	config = new IndexWriterConfig(Version.LUCENE_41, ANALYZER);
	this.wikiIdtToLuceneIdSerialization = wikiIdtToLuceneIdSerialization;

	BooleanQuery.setMaxClauseCount(1000);

	try {
		index = FSDirectory.open(indexPath);
		// writer.commit();
	} catch (Exception e) {
		logger.error("opening the index: {}", e.toString());
		System.exit(1);
	}

	summarizer = new ArticleSummarizer();
	writer = getWriter();
	collectionSize = writer.numDocs();
	wikiIdToLuceneId = Collections.emptyMap();
}
 
Example 15
Source Project: bluima   Source File: Txt2PubmedIdIndexer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(UimaContext context)
        throws ResourceInitializationException {
    super.initialize(context);
    try {
        // create writer
        Directory dir;
        dir = FSDirectory.open(new File(INDEX_PATH));
        Analyzer analyzer = getAnalyzer();
        IndexWriterConfig iwc = new IndexWriterConfig(
                Version.LUCENE_41, analyzer);
        iwc.setOpenMode(OpenMode.CREATE);
        indexWriter = new IndexWriter(dir, iwc);
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 16
Source Project: tagme   Source File: AnchorIndexer.java    License: Apache License 2.0 6 votes vote down vote up
private IndexSearcher openWikipediaIndex(String lang) throws IOException{
	
	
	File indexDir = RepositoryDirs.WIKIPEDIA.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	if (indexSize < maxMemory * GAP_FACTOR){
		
		log.info("MaxMemory is enough, loading Wikipedia index...");
		IndexReader r = IndexReader.open(new RAMDirectory(FSDirectory.open(indexDir)), true);
		log.info("WikipediaIndex loaded.");
		return new IndexSearcher(r);
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load WikipediaIndex (about "+indexSize/1000000+"Mb)");
		return Indexes.getSearcher(RepositoryDirs.WIKIPEDIA.getPath(lang));
	}
}
 
Example 17
Source Project: incubator-retired-blur   Source File: BlockDirectory.java    License: Apache License 2.0 6 votes vote down vote up
private long getFileModified(String name) throws IOException {
  if (_directory instanceof FSDirectory) {
    File directory = ((FSDirectory) _directory).getDirectory();
    File file = new File(directory, name);
    if (!file.exists()) {
      throw new FileNotFoundException("File [" + name + "] not found");
    }
    return file.lastModified();
  } else if (_directory instanceof HdfsDirectory) {
    return ((HdfsDirectory) _directory).getFileModified(name);
  } else if (_directory instanceof LastModified) {
    return ((LastModified) _directory).getFileModified(name);
  } else {
    throw new RuntimeException("Not supported");
  }
}
 
Example 18
Source Project: webdsl   Source File: AbstractIndexManager.java    License: Apache License 2.0 6 votes vote down vote up
protected static boolean clearIndex(File path) {
	try {
		if (path == null || !path.exists())
			return true; // if path doesnt exist, then there is nothing to
							// clear

		FSDirectory indexDir = new FSDirectoryProvider().getDirectory();
		IndexWriter writer = new IndexWriter(indexDir.open(path),
				new IndexWriterConfig(Version.LUCENE_CURRENT,
						new WhitespaceAnalyzer(Version.LUCENE_CURRENT)));
		writer.deleteAll();
		writer.close();
		return true;
	} catch (Exception ex) {
		org.webdsl.logging.Logger.error(
				"Error while clearing index on location: " + path, ex);
		return false;
	}

}
 
Example 19
Source Project: lucene-solr   Source File: TestUtil.java    License: Apache License 2.0 6 votes vote down vote up
/** Returns true if VirusCheckingFS is in use and was in fact already enabled */
public static boolean disableVirusChecker(Directory in) {
  Directory dir = FilterDirectory.unwrap(in);
  if (dir instanceof FSDirectory) {

    FileSystem fs = ((FSDirectory) dir).getDirectory().getFileSystem();
    while (fs instanceof FilterFileSystem) {
      FilterFileSystem ffs = (FilterFileSystem) fs;
      if (ffs.getParent() instanceof VirusCheckingFS) {
        VirusCheckingFS vfs = (VirusCheckingFS) ffs.getParent();
        boolean isEnabled = vfs.isEnabled();
        vfs.disable();
        return isEnabled;
      }
      fs = ffs.getDelegate();
    }
  }

  return false;
}
 
Example 20
Source Project: tephra   Source File: LuceneHelperImpl.java    License: MIT License 6 votes vote down vote up
private synchronized Directory get(String key) {
    if (map == null) {
        map = new ConcurrentHashMap<>();
        BooleanQuery.setMaxClauseCount(Integer.MAX_VALUE);
    }

    return map.computeIfAbsent(key, k -> {
        Path path = Paths.get(context.getAbsoluteRoot(), root, k, "index");
        io.mkdirs(path.toFile());
        if (logger.isInfoEnable())
            logger.info("设置Lucene索引根目录[{}:{}]。", k, path);
        try {
            return FSDirectory.open(path);
        } catch (IOException e) {
            logger.warn(e, "打开Lucene索引目录[{}:{}]时发生异常!", k, path);

            return null;
        }
    });
}
 
Example 21
public VectorStoreReaderLucene(String vectorFileName, FlagConfig flagConfig) throws IOException {
  this.flagConfig = flagConfig;
  this.vectorFileName = vectorFileName;
  this.vectorFile = new File(vectorFileName);
  try {
    String parentPath = this.vectorFile.getParent();
    if (parentPath == null) parentPath = "";
    this.directory = FSDirectory.open(FileSystems.getDefault().getPath(parentPath));  // Old from FSDirectory impl.
    // Read number of dimension from header information.
    this.threadLocalIndexInput = new ThreadLocal<IndexInput>() {
      @Override
      protected IndexInput initialValue() {
        try {
          return directory.openInput(vectorFile.getName(), IOContext.READ);
        } catch (IOException e) {
          throw new RuntimeException(e.getMessage(), e);
        }
      }
    };
    readHeadersFromIndexInput(flagConfig);
  } catch (IOException e) {
    logger.warning("Cannot open file: " + this.vectorFileName + "\n" + e.getMessage());
    throw e;
  }
}
 
Example 22
Source Project: bioasq   Source File: LuceneDocumentRetrievalExecutor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 100);
  // query constructor
  constructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          BooleanBagOfPhraseQueryStringConstructor.class, QueryStringConstructor.class);
  // lucene
  Analyzer analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  String[] fields = UimaContextHelper.getConfigParameterStringArrayValue(context, "fields");
  parser = new MultiFieldQueryParser(fields, analyzer);
  String index = UimaContextHelper.getConfigParameterStringValue(context, "index");
  try {
    reader = DirectoryReader.open(FSDirectory.open(Paths.get(index)));
  } catch (IOException e) {
    throw new ResourceInitializationException(e);
  }
  searcher = new IndexSearcher(reader);
  idFieldName = UimaContextHelper.getConfigParameterStringValue(context, "id-field", null);
  titleFieldName = UimaContextHelper.getConfigParameterStringValue(context, "title-field", null);
  textFieldName = UimaContextHelper.getConfigParameterStringValue(context, "text-field", null);
  uriPrefix = UimaContextHelper.getConfigParameterStringValue(context, "uri-prefix", null);
}
 
Example 23
Source Project: RedisDirectory   Source File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testMMapDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    FSDirectory open = FSDirectory.open(Paths.get("E:/testlucene"));
    IndexWriter indexWriter = new IndexWriter(open, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("MMapDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(open));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("MMapDirectory search consumes {}ms!", (end - start));
}
 
Example 24
@Override
public boolean delete(SearchDto searchDto) throws IOException {
	IndexWriter writer = null;
	try {
		IndexWriterConfig conf = new IndexWriterConfig(new StandardAnalyzer());
		conf.setOpenMode(OpenMode.CREATE_OR_APPEND);
		writer = new IndexWriter(FSDirectory.open(Paths.get(settingCache.get(ISetting.S_LUCENE_DIR).getValue())), conf);
		writer.deleteDocuments(new Term(ID, searchDto.getId()));
	} catch (Exception e) {
		e.printStackTrace();
		stringCache.add(IConst.C_CACHE_ERROR_TIP, "Lucene删除异常,请联系管理员查看日志,错误信息:" + e.getMessage());
	} finally {
		if (writer != null) {
			writer.close();
		}
	}
	return true;
}
 
Example 25
Source Project: SourcererCC   Source File: CodeSearcher.java    License: GNU General Public License v3.0 6 votes vote down vote up
public CodeSearcher(String indexDir, String field) {
    logger.info("index directory: "+ indexDir);
    this.field = field;
    this.indexDir = indexDir;
    try {
        this.reader = DirectoryReader.open(FSDirectory.open(new File(
                this.indexDir)));
    } catch (IOException e) {
        logger.error("cant get the reader to index dir, exiting, "
                + indexDir);
        e.printStackTrace();
        System.exit(1);
    }
    this.searcher = new IndexSearcher(this.reader);
    this.analyzer = new KeywordAnalyzer();//
            //new WhitespaceAnalyzer(Version.LUCENE_46); // TODO: pass
                                                               // the
                                                               // analyzer
                                                               // as
                                                               // argument
                                                               // to
                                                               // constructor
    new CloneHelper(); // i don't remember why we are making this object?
    this.queryParser = new QueryParser(Version.LUCENE_46, this.field,
            analyzer);
}
 
Example 26
Source Project: ES-Fastloader   Source File: AppendLuceneTransportAction.java    License: Apache License 2.0 5 votes vote down vote up
private void doExecuteCore(AppendLuceneRequest request, ActionListener<AppendLuceneResponse> listener) {
    try {
        // 对请求做check
        request.check();

        // 获得shard信息
        ShardId shardId = new ShardId(request.indexName, request.uuid, request.shardId);
        IndexShard shard = indicesService.getShardOrNull(shardId);
        if(shard==null) {
            throw new Exception("shard not found, indexName:" + request.indexName + ", shardId:" + request.shardId);
        }

        // 获得lucene的IndexWriter对象
        /* FIXME 这里需要修改es的代码, 将lucene的IndexWriter对象暴露给plugin使用  */
        InternalEngine engine = (InternalEngine) shard.getEngineOrNull();
        IndexWriter indexWriter = engine.getIndexWriter();

        // 处理主键冲突情况
        long deleteCount = -1;
        List<String> appendDirs = request.getAppendDirs();
        if(request.primeKey!=null && request.primeKey.length()>0) {
            deleteCount = doPrimerKey(appendDirs, indexWriter, request.primeKey);
        }

        // 将新的lucene文件加入到shard中
        Directory[] indexes = new Directory[appendDirs.size()];
        for(int i=0; i<appendDirs.size(); i++) {
            indexes[i] = FSDirectory.open(Paths.get(appendDirs.get(i)));
        }
        indexWriter.addIndexes(indexes);
        indexWriter.commit();

        // 构建response
        AppendLuceneResponse response = new AppendLuceneResponse();
        response.deleteCount = deleteCount;
        listener.onResponse(response);
    } catch (Exception e) {
        listener.onFailure(e);
    }
}
 
Example 27
Source Project: lucene-geo-gazetteer   Source File: GeoNameResolver.java    License: Apache License 2.0 5 votes vote down vote up
private IndexReader createIndexReader(String indexerPath) throws IOException {
	File indexfile = new File(indexerPath);
	indexDir = FSDirectory.open(indexfile.toPath());


	if (!DirectoryReader.indexExists(indexDir)) {
		LOG.log(Level.SEVERE,
				"No Lucene Index Dierctory Found, Invoke indexBuild() First !");
		System.exit(1);
	}

	return DirectoryReader.open(indexDir);
}
 
Example 28
Source Project: nifi   Source File: StandardIndexManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public EventIndexSearcher borrowIndexSearcher(final File indexDir) throws IOException {
    final File absoluteFile = indexDir.getAbsoluteFile();

    final IndexWriterCount writerCount;
    synchronized (writerCounts) {
        writerCount = writerCounts.remove(absoluteFile);

        if (writerCount != null) {
            // Increment writer count and create an Index Searcher based on the writer
            writerCounts.put(absoluteFile, new IndexWriterCount(writerCount.getWriter(), writerCount.getAnalyzer(),
                writerCount.getDirectory(), writerCount.getCount() + 1, writerCount.isCloseableWhenUnused()));
        }
    }

    final DirectoryReader directoryReader;
    if (writerCount == null) {
        logger.trace("Creating index searcher for {}", indexDir);
        final Directory directory = FSDirectory.open(indexDir.toPath());
        directoryReader = DirectoryReader.open(directory);
    } else {
        final EventIndexWriter eventIndexWriter = writerCount.getWriter();
        directoryReader = DirectoryReader.open(eventIndexWriter.getIndexWriter(), false, false);
    }

    final IndexSearcher searcher = new IndexSearcher(directoryReader, this.searchExecutor);

    logger.trace("Created index searcher {} for {}", searcher, indexDir);
    return new LuceneEventIndexSearcher(searcher, indexDir, null, directoryReader);
}
 
Example 29
Source Project: taoshop   Source File: SearchBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public static void doSearch(String indexDir , String queryStr) throws IOException, ParseException, InvalidTokenOffsetsException {
    Directory directory = FSDirectory.open(Paths.get(indexDir));
    DirectoryReader reader = DirectoryReader.open(directory);
    IndexSearcher searcher = new IndexSearcher(reader);
    Analyzer analyzer = new SmartChineseAnalyzer();
    QueryParser parser = new QueryParser("tcontent",analyzer);
    Query query = parser.parse(queryStr);

    long startTime = System.currentTimeMillis();
    TopDocs docs = searcher.search(query,10);

    System.out.println("查找"+queryStr+"所用时间:"+(System.currentTimeMillis()-startTime));
    System.out.println("查询到"+docs.totalHits+"条记录");

    //加入高亮显示的
    SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<b><font color=red>","</font></b>");
    QueryScorer scorer = new QueryScorer(query);//计算查询结果最高的得分
    Fragmenter fragmenter = new SimpleSpanFragmenter(scorer);//根据得分算出一个片段
    Highlighter highlighter = new Highlighter(simpleHTMLFormatter,scorer);
    highlighter.setTextFragmenter(fragmenter);//设置显示高亮的片段

    //遍历查询结果
    for(ScoreDoc scoreDoc : docs.scoreDocs){
        Document doc = searcher.doc(scoreDoc.doc);
        System.out.println(doc.get("title"));
        System.out.println(doc.get("tcontent"));
        String tcontent = doc.get("tcontent");
        if(tcontent != null){
            TokenStream tokenStream =  analyzer.tokenStream("tcontent", new StringReader(tcontent));
            String summary = highlighter.getBestFragment(tokenStream, tcontent);
            System.out.println(summary);
        }
    }
    reader.close();
}
 
Example 30
Source Project: Siamese   Source File: Siamese.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * Read the ES index file
 * @param indexName the name of the index
 */
private void readESIndex(String indexName) {
    String indexFile = elasticsearchLoc + "/data/stackoverflow/nodes/0/indices/"
            + indexName + "/0/index";
    try {
        esIndexRader = DirectoryReader.open(FSDirectory.open(Paths.get(indexFile)));
    } catch (IOException e) {
        e.printStackTrace();
    }
}