org.apache.lucene.store.SimpleFSDirectory Java Examples

The following examples show how to use org.apache.lucene.store.SimpleFSDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NodeEnvironment.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Acquires, then releases, all {@code write.lock} files in the given
 * shard paths. The "write.lock" file is assumed to be under the shard
 * path's "index" directory as used by Elasticsearch.
 *
 * @throws LockObtainFailedException if any of the locks could not be acquired
 */
public static void acquireFSLockForPaths(Settings indexSettings, Path... shardPaths) throws IOException {
    Lock[] locks = new Lock[shardPaths.length];
    Directory[] dirs = new Directory[shardPaths.length];
    try {
        for (int i = 0; i < shardPaths.length; i++) {
            // resolve the directory the shard actually lives in
            Path p = shardPaths[i].resolve("index");
            // open a directory (will be immediately closed) on the shard's location
            dirs[i] = new SimpleFSDirectory(p, FsDirectoryService.buildLockFactory(indexSettings));
            // create a lock for the "write.lock" file
            try {
                locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
            } catch (IOException ex) {
                throw new LockObtainFailedException("unable to acquire " +
                        IndexWriter.WRITE_LOCK_NAME + " for " + p);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(locks);
        IOUtils.closeWhileHandlingException(dirs);
    }
}
 
Example #2
Source File: NodeEnvironment.java    From crate with Apache License 2.0 6 votes vote down vote up
/**
 * Acquires, then releases, all {@code write.lock} files in the given
 * shard paths. The "write.lock" file is assumed to be under the shard
 * path's "index" directory as used by Elasticsearch.
 *
 * @throws LockObtainFailedException if any of the locks could not be acquired
 */
public static void acquireFSLockForPaths(IndexSettings indexSettings, Path... shardPaths) throws IOException {
    Lock[] locks = new Lock[shardPaths.length];
    Directory[] dirs = new Directory[shardPaths.length];
    try {
        for (int i = 0; i < shardPaths.length; i++) {
            // resolve the directory the shard actually lives in
            Path p = shardPaths[i].resolve("index");
            // open a directory (will be immediately closed) on the shard's location
            dirs[i] = new SimpleFSDirectory(p, indexSettings.getValue(FsDirectoryService.INDEX_LOCK_FACTOR_SETTING));
            // create a lock for the "write.lock" file
            try {
                locks[i] = dirs[i].obtainLock(IndexWriter.WRITE_LOCK_NAME);
            } catch (IOException ex) {
                throw new LockObtainFailedException("unable to acquire " +
                                IndexWriter.WRITE_LOCK_NAME + " for " + p, ex);
            }
        }
    } finally {
        IOUtils.closeWhileHandlingException(locks);
        IOUtils.closeWhileHandlingException(dirs);
    }
}
 
Example #3
Source File: Indexes.java    From tagme with Apache License 2.0 6 votes vote down vote up
public static IndexReader getReader(String path) throws IOException
{
	if (!readerMap.containsKey(path))
	{
		synchronized(Indexes.class)
		{
			if (!readerMap.containsKey(path))
			{
				log.info("["+path+"] Opening...");
				if (! new File(path).exists())
					throw new ConfigurationException("Unable to find index in "+path);

				IndexReader r = IndexReader.open(new SimpleFSDirectory(new File(path)), true);
				readerMap.put(path, r);
				log.info("["+path+"] Opened. Memory: "+ExternalSortUtils.memSize(false));
			}
		}
	}
	
	return readerMap.get(path);
}
 
Example #4
Source File: IndexManager.java    From dependency-track with Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the index directory based on the type of index used.
 * @return a Directory
 * @throws IOException when the directory cannot be accessed
 * @since 3.0.0
 */
private synchronized Directory getDirectory() throws IOException {
    final File indexDir = getIndexDirectory(indexType);
    if (!indexDir.exists()) {
        if (!indexDir.mkdirs()) {
            LOGGER.error("Unable to create index directory: " + indexDir.getCanonicalPath());
            Notification.dispatch(new Notification()
                    .scope(NotificationScope.SYSTEM)
                    .group(NotificationGroup.FILE_SYSTEM)
                    .title(NotificationConstants.Title.FILE_SYSTEM_ERROR)
                    .content("Unable to create index directory: " + indexDir.getCanonicalPath())
                    .level(NotificationLevel.ERROR)
            );
        }
    }
    return new SimpleFSDirectory(indexDir.toPath());
}
 
Example #5
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 5 votes vote down vote up
@Transactional(readOnly = true)
public void deleteIndex(Integer contentId) throws IOException,
		ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	deleteIndex(contentId, dir);
}
 
Example #6
Source File: Store.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Tries to open an index for the given location. This includes reading the
 * segment infos and possible corruption markers. If the index can not
 * be opened, an exception is thrown
 */
public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
    try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5));
         Directory dir = new SimpleFSDirectory(indexLocation)) {
        failIfCorrupted(dir, shardId);
        SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
        logger.trace("{} loaded segment info [{}]", shardId, segInfo);
    }
}
 
Example #7
Source File: FsDirectoryService.java    From crate with Apache License 2.0 5 votes vote down vote up
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
    final String storeType = indexSettings.getSettings()
        .get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(
            IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    switch (type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, mMapDirectory);
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return new MMapDirectory(location, lockFactory);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}
 
Example #8
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 5 votes vote down vote up
@Transactional(readOnly = true)
public List<Content> searchList(String path, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int first, int max) throws CorruptIndexException, IOException,
		ParseException {
	Directory dir = new SimpleFSDirectory(new File(path));
	return searchList(dir, queryString,category,workplace, siteId, channelId, startDate,
			endDate, first, max);
}
 
Example #9
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 5 votes vote down vote up
@Transactional(readOnly = true)
public Pagination searchPage(String path, String queryString,String category,String workplace,
		Integer siteId, Integer channelId, Date startDate, Date endDate,
		int pageNo, int pageSize) throws CorruptIndexException,
		IOException, ParseException {
	Directory dir = new SimpleFSDirectory(new File(path));
	return searchPage(dir, queryString, category,workplace,siteId, channelId, startDate,
			endDate, pageNo, pageSize);
}
 
Example #10
Source File: NexusRepositoryIndexerImpl.java    From netbeans with Apache License 2.0 5 votes vote down vote up
@Override
public List<RepositoryInfo> getLoaded(final List<RepositoryInfo> repos) {
    final List<RepositoryInfo> toRet = new ArrayList<RepositoryInfo>(repos.size());
    for (final RepositoryInfo repo : repos) {
        File loc = new File(getDefaultIndexLocation(), repo.getId()); // index folder
        try {
            if (loc.exists() && new File(loc, "timestamp").exists() && DirectoryReader.indexExists(new SimpleFSDirectory(loc.toPath()))) {
                toRet.add(repo);
            }
        } catch (IOException ex) {
            LOGGER.log(Level.FINER, "Index Not Available: " +repo.getId() + " at: " + loc.getAbsolutePath(), ex);
        }
    }
    return toRet;
}
 
Example #11
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 5 votes vote down vote up
@Transactional(readOnly = true)
public Integer createIndex(Integer siteId, Integer channelId,
		Date startDate, Date endDate, Integer startId, Integer max)
		throws IOException, ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	return createIndex(siteId, channelId, startDate, endDate, startId, max,
			dir);
}
 
Example #12
Source File: MultiDataPathUpgrader.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Runs check-index on the target shard and throws an exception if it failed
 */
public void checkIndex(ShardPath targetPath) throws IOException {
    BytesStreamOutput os = new BytesStreamOutput();
    PrintStream out = new PrintStream(os, false, Charsets.UTF_8.name());
    try (Directory directory = new SimpleFSDirectory(targetPath.resolveIndex());
        final CheckIndex checkIndex = new CheckIndex(directory)) {
        checkIndex.setInfoStream(out);
        CheckIndex.Status status = checkIndex.checkIndex();
        out.flush();
        if (!status.clean) {
            logger.warn("check index [failure]\n{}", new String(os.bytes().toBytes(), Charsets.UTF_8));
            throw new IllegalStateException("index check failure");
        }
    }
}
 
Example #13
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 4 votes vote down vote up
@Transactional(readOnly = true)
public void createIndex(Content content) throws IOException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	createIndex(content, dir);
}
 
Example #14
Source File: LuceneContentSvcImpl.java    From Lottery with GNU General Public License v2.0 4 votes vote down vote up
public void updateIndex(Content content) throws IOException, ParseException {
	String path = realPathResolver.get(Constants.LUCENE_PATH);
	Directory dir = new SimpleFSDirectory(new File(path));
	updateIndex(content, dir);
}
 
Example #15
Source File: SearcherDAO.java    From entando-core with GNU Lesser General Public License v3.0 4 votes vote down vote up
private IndexSearcher getSearcher() throws IOException {
    FSDirectory directory = new SimpleFSDirectory(indexDir.toPath());
    IndexReader reader = DirectoryReader.open(directory);
    IndexSearcher searcher = new IndexSearcher(reader);
    return searcher;
}
 
Example #16
Source File: TopicIndexer.java    From tagme with Apache License 2.0 4 votes vote down vote up
@Override
	public void makeIndex(String lang, File workingDir) throws IOException
	{
		
		IndexReader articles = Indexes.getReader(RepositoryDirs.WIKIPEDIA.getPath(lang));
		Int2ObjectMap<String> bestAnchorMap = new BestAnchors(lang).getDataset();
		
		IndexWriter index = new IndexWriter(new SimpleFSDirectory(workingDir), new IndexWriterConfig(Version.LUCENE_34, new KeywordAnalyzer()));
		Document doc = new Document();
		Field fWID = new Field(FIELD_WID, "", Store.YES, Index.NOT_ANALYZED);
		Field fTitle = new Field(FIELD_TITLE, "", Store.YES, Index.NOT_ANALYZED);
		Field fAbstract = new Field(FIELD_ABSTRACT, "", Store.YES, Index.NO);
		Field fBestAnchor = new Field(FIELD_BEST_ANCHOR, "", Store.YES, Index.NO);
		doc.add(fWID);
		doc.add(fTitle);
		doc.add(fAbstract);
		doc.add(fBestAnchor);
				
		
		int max = articles.maxDoc();
		PLogger plog = new PLogger(log, Step.TEN_MINUTES, "pages", "indexed", "noBest");
		plog.setEnd(max);
		plog.start("Start indexing...");
		
		for(int i=0; i<max; i++)
		{
			plog.update(0);
			Document oldDoc = articles.document(i);
			PageType type = PageType.valueOf(oldDoc.get(WikipediaIndexer.FIELD_TYPE));
			if (type == PageType.TOPIC)
			{
				int wid = Integer.parseInt(oldDoc.get(WikipediaIndexer.FIELD_WID));
				fWID.setValue(oldDoc.get(WikipediaIndexer.FIELD_WID));
				fAbstract.setValue(oldDoc.get(WikipediaIndexer.FIELD_ABSTRACT));
				fTitle.setValue(oldDoc.get(WikipediaIndexer.FIELD_TITLE));
				
				String bestAnchor = bestAnchorMap.get(wid);
				if (bestAnchor == null || bestAnchor.length() == 0) plog.update(2);
				fBestAnchor.setValue(bestAnchor==null?"":bestAnchor);
				
				String[] cats = oldDoc.getValues(WikipediaIndexer.FIELD_CAT);
				if (cats != null) {
					for (int j=0; j<cats.length; j++)
						doc.add(new Field(FIELD_CAT, cats[j], Store.YES, Index.NOT_ANALYZED));
				}
				
				index.addDocument(doc);
				plog.update(1);
				
				doc.removeFields(FIELD_CAT);
			}
		}
		
		plog.stop();
		
		log.info("Now optimizing...");
		index.optimize();
		
		index.close();
		
		//we cannot call this because the index is still in the temporary dir
		//so TopicDocs will be created using old index
//		log.info("Index Done, now creating WID->DOC_ID map");
//		
//		TopicDocs td = new TopicDocs(lang);
//		td.forceParsing();
		
		log.info("Done.");
	}
 
Example #17
Source File: HunspellService.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Loads the hunspell dictionary for the given local.
 *
 * @param locale       The locale of the hunspell dictionary to be loaded.
 * @param nodeSettings The node level settings
 * @param env          The node environment (from which the conf path will be resolved)
 * @return The loaded Hunspell dictionary
 * @throws Exception when loading fails (due to IO errors or malformed dictionary files)
 */
private Dictionary loadDictionary(String locale, Settings nodeSettings, Environment env) throws Exception {
    if (LOGGER.isDebugEnabled()) {
        LOGGER.debug("Loading hunspell dictionary [{}]...", locale);
    }
    Path dicDir = hunspellDir.resolve(locale);
    if (FileSystemUtils.isAccessibleDirectory(dicDir, LOGGER) == false) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Could not find hunspell dictionary [%s]", locale));
    }

    // merging node settings with hunspell dictionary specific settings
    Settings dictSettings = HUNSPELL_DICTIONARY_OPTIONS.get(nodeSettings);
    nodeSettings = loadDictionarySettings(dicDir, dictSettings.getByPrefix(locale + "."));

    boolean ignoreCase = nodeSettings.getAsBoolean("ignore_case", defaultIgnoreCase);

    Path[] affixFiles = FileSystemUtils.files(dicDir, "*.aff");
    if (affixFiles.length == 0) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Missing affix file for hunspell dictionary [%s]", locale));
    }
    if (affixFiles.length != 1) {
        throw new ElasticsearchException(String.format(Locale.ROOT, "Too many affix files exist for hunspell dictionary [%s]", locale));
    }
    InputStream affixStream = null;

    Path[] dicFiles = FileSystemUtils.files(dicDir, "*.dic");
    List<InputStream> dicStreams = new ArrayList<>(dicFiles.length);
    try {

        for (int i = 0; i < dicFiles.length; i++) {
            dicStreams.add(Files.newInputStream(dicFiles[i]));
        }

        affixStream = Files.newInputStream(affixFiles[0]);

        try (Directory tmp = new SimpleFSDirectory(env.tmpFile())) {
            return new Dictionary(tmp, "hunspell", affixStream, dicStreams, ignoreCase);
        }

    } catch (Exception e) {
        LOGGER.error(() -> new ParameterizedMessage("Could not load hunspell dictionary [{}]", locale), e);
        throw e;
    } finally {
        IOUtils.close(affixStream);
        IOUtils.close(dicStreams);
    }
}
 
Example #18
Source File: AnchorTernaryTrieDump.java    From tagme with Apache License 2.0 4 votes vote down vote up
@Override
protected AnchorTernaryTrie parseSet() throws IOException
{
	
	File indexDir = RepositoryDirs.ANCHORS.getDir(lang);
	long indexSize = FileUtils.sizeOfDirectory(indexDir);
	long maxMemory = Runtime.getRuntime().maxMemory();
	
	IndexReader anchors;
	if (indexSize < maxMemory * 0.8){
		
		log.info("MaxMemory is enough, loading Anchor index...");
		anchors = IndexReader.open(new RAMDirectory(new SimpleFSDirectory(indexDir)), true);
		log.info("Anchor index loaded.");
		
	} else {
		log.info("Not enough memory ["+maxMemory/1000000+"Mb] to load Anchor index (about "+indexSize/1000000+"Mb)");
		anchors = Indexes.getReader(RepositoryDirs.ANCHORS.getPath(lang));
	}

	
	AnchorTernaryTrie trie = new AnchorTernaryTrie();
	
	int maxdoc = anchors.maxDoc();
	
	IntList doclist = new IntArrayList();
	for(int i=0;i<maxdoc;i++) doclist.add(i);
	Random rnd = new Random(System.currentTimeMillis());
	
	PLogger plog = new PLogger(log, Step.TEN_MINUTES, "anchors", "skipped", "duplicates");
	plog.setEnd(0, maxdoc);
	plog.start("Inserting in to trie...");
	while(!doclist.isEmpty())
	{
		int docID = doclist.removeInt(rnd.nextInt(doclist.size()));
		
		plog.update(0);
		Document doc = anchors.document(docID);
		if (doc == null){
			plog.update(1);
			continue;
		}
		
		String anchorText = doc.get(AnchorIndexer.FIELD_TEXT);
		String serial = doc.get(AnchorIndexer.FIELD_OBJECT);
		Anchor anchorObj = Anchor.deserialize(serial);
		
		if (anchorObj == null){
			plog.update(1);
			continue;
		}
		
		boolean added = trie.add(anchorText, anchorObj);
		
		if (!added) plog.update(2);
	}
	plog.stop();
	
	return trie;
}
 
Example #19
Source File: MetaDataStateFormat.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
protected Directory newDirectory(Path dir) throws IOException {
    return new SimpleFSDirectory(dir);
}
 
Example #20
Source File: MetaDataStateFormat.java    From crate with Apache License 2.0 4 votes vote down vote up
protected Directory newDirectory(Path dir) throws IOException {
    return new SimpleFSDirectory(dir);
}
 
Example #21
Source File: IndexStoreTests.java    From crate with Apache License 2.0 4 votes vote down vote up
private void doTestStoreDirectory(Index index,
                                  Path tempDir,
                                  String typeSettingValue,
                                  IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder()
        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryService service = new FsDirectoryService(
        indexSettings, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
        switch (type) {
            case HYBRIDFS:
                assertHybridDirectory(directory);
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertHybridDirectory(directory);
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}