org.apache.lucene.store.NIOFSDirectory Java Examples

The following examples show how to use org.apache.lucene.store.NIOFSDirectory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: LuceneExample.java    From yuzhouwan with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    // index
    try (Directory index = new NIOFSDirectory(Paths.get("/tmp/index"))) {
        // add
        try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer()))) {
            Document doc = new Document();
            doc.add(new TextField("blog", "yuzhouwan.com", Field.Store.YES));
            doc.add(new StringField("github", "asdf2014", Field.Store.YES));
            writer.addDocument(doc);
            writer.commit();
        }
        // search
        try (DirectoryReader reader = DirectoryReader.open(index)) {
            IndexSearcher searcher = new IndexSearcher(reader);
            QueryParser parser = new QueryParser("blog", new StandardAnalyzer());
            Query query = parser.parse("yuzhouwan.com");
            ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
            for (ScoreDoc hit : hits) {
                Document hitDoc = searcher.doc(hit.doc);
                System.out.println(hitDoc.get("blog"));
            }
        }
    }
}
 
Example #2
Source File: LuceneIndexNIOFS.java    From rdf4j with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
@Override
protected Directory createDirectory(Properties parameters) throws IOException {
	if (parameters.containsKey(LuceneSail.LUCENE_DIR_KEY)) {
		return new NIOFSDirectory(Paths.get(parameters.getProperty(LuceneSail.LUCENE_DIR_KEY)));
	} else {
		return super.createDirectory(parameters);
	}
}
 
Example #3
Source File: LuceneSearcher.java    From jpress with GNU Lesser General Public License v3.0 5 votes vote down vote up
public LuceneSearcher() {
    File indexDir = new File(PathKit.getRootClassPath(), INDEX_PATH);
    if (!indexDir.exists()) {
        indexDir.mkdirs();
    }
    try {
        directory = NIOFSDirectory.open(indexDir.toPath());
    } catch (Exception e) {
        LOG.error(e.toString(), e);
    }
}
 
Example #4
Source File: LuceneSearcher.java    From jpress with GNU Lesser General Public License v3.0 5 votes vote down vote up
public LuceneSearcher() {
    File indexDir = new File(PathKit.getRootClassPath(), INDEX_PATH);
    if (!indexDir.exists()) {
        indexDir.mkdirs();
    }
    try {
        directory = NIOFSDirectory.open(indexDir.toPath());
    } catch (Exception e) {
        LOG.error(e.toString(), e);
    }
}
 
Example #5
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testPendingDeletionsRollbackWithReader() throws IOException {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  try (FSDirectory _dir = new NIOFSDirectory(root)) {
    Directory dir = new FilterDirectory(_dir) {};

    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    Document d = new Document();
    d.add(new StringField("id", "1", Field.Store.YES));
    d.add(new NumericDocValuesField("numval", 1));
    w.addDocument(d);
    w.commit();
    w.addDocument(d);
    w.flush();
    DirectoryReader reader = DirectoryReader.open(w);
    w.rollback();

    // try-delete superfluous files (some will fail due to windows-fs)
    IndexWriterConfig iwc2 = new IndexWriterConfig(new MockAnalyzer(random()));
    new IndexWriter(dir, iwc2).close();

    // test that we can index on top of pending deletions
    IndexWriterConfig iwc3 = new IndexWriterConfig(new MockAnalyzer(random()));
    w = new IndexWriter(dir, iwc3);
    w.addDocument(d);
    w.commit();

    reader.close();
    w.close();
  }
}
 
Example #6
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testPendingDeletesAlreadyWrittenFiles() throws IOException {
  Path path = createTempDir();
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  DirectoryReader reader;
  // MMapDirectory doesn't work because it closes its file handles after mapping!
  try (FSDirectory dir = new NIOFSDirectory(root)) {
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    IndexWriter w = new IndexWriter(dir, iwc);
    w.commit();
    IndexInput in = dir.openInput("segments_1", IOContext.DEFAULT);
    w.addDocument(new Document());
    w.close();

    assertTrue(dir.getPendingDeletions().size() > 0);

    // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
    expectThrows(NoSuchFileException.class, () -> {
      dir.deleteFile("segments_1");
    });
    new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random()))).close();
    in.close();
  }
}
 
Example #7
Source File: FsDirectoryService.java    From crate with Apache License 2.0 5 votes vote down vote up
protected Directory newFSDirectory(Path location, LockFactory lockFactory) throws IOException {
    final String storeType = indexSettings.getSettings()
        .get(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), IndexModule.Type.FS.getSettingsKey());
    IndexModule.Type type;
    if (IndexModule.Type.FS.match(storeType)) {
        type = IndexModule.defaultStoreType(
            IndexModule.NODE_STORE_ALLOW_MMAP.getWithFallback(indexSettings.getNodeSettings()));
    } else {
        type = IndexModule.Type.fromSettingsKey(storeType);
    }
    switch (type) {
        case HYBRIDFS:
            // Use Lucene defaults
            final FSDirectory primaryDirectory = FSDirectory.open(location, lockFactory);
            if (primaryDirectory instanceof MMapDirectory) {
                MMapDirectory mMapDirectory = (MMapDirectory) primaryDirectory;
                return new HybridDirectory(lockFactory, mMapDirectory);
            } else {
                return primaryDirectory;
            }
        case MMAPFS:
            return new MMapDirectory(location, lockFactory);
        case SIMPLEFS:
            return new SimpleFSDirectory(location, lockFactory);
        case NIOFS:
            return new NIOFSDirectory(location, lockFactory);
        default:
            throw new AssertionError("unexpected built-in store type [" + type + "]");
    }
}
 
Example #8
Source File: LuceneCorpusAdapter.java    From Palmetto with GNU Affero General Public License v3.0 5 votes vote down vote up
/**
 * Creates a corpus adapter which uses the Lucene index with the given path
 * and searches on the field with the given field name.
 * 
 * @param indexPath
 * @param fieldName
 * @return
 * @throws CorruptIndexException
 * @throws IOException
 */
public static LuceneCorpusAdapter create(String indexPath, String fieldName)
        throws CorruptIndexException, IOException {
    DirectoryReader dirReader = DirectoryReader.open(new NIOFSDirectory(new File(indexPath)));
    List<AtomicReaderContext> leaves = dirReader.leaves();
    AtomicReader reader[] = new AtomicReader[leaves.size()];
    AtomicReaderContext contexts[] = new AtomicReaderContext[leaves.size()];
    for (int i = 0; i < reader.length; i++) {
        contexts[i] = leaves.get(i);
        reader[i] = contexts[i].reader();
    }
    return new LuceneCorpusAdapter(dirReader, reader, contexts, fieldName);
}
 
Example #9
Source File: LuceneStoreImpl.java    From karaf-decanter with Apache License 2.0 5 votes vote down vote up
@Activate
public void activate() throws Exception {
    directory = new NIOFSDirectory(Paths.get(System.getProperty("karaf.data"), INDEX_DIRECTORY));
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new StandardAnalyzer());
    indexWriter = new IndexWriter(directory, indexWriterConfig);
    points = loadPoints();
}
 
Example #10
Source File: IndexBuilder.java    From exhibitor with Apache License 2.0 5 votes vote down vote up
public void open() throws Exception
{
    if ( !directory.exists() && !directory.mkdirs() )
    {
        throw new IOException("Could not make: " + directory);
    }

    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    niofsDirectory = new NIOFSDirectory(directory, new SingleInstanceLockFactory());
    writer = new IndexWriter(niofsDirectory, conf);
}
 
Example #11
Source File: TestReplicationHandlerBackup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void verify(Path backup, int nDocs) throws IOException {
  log.info("Verifying ndocs={} in {}", nDocs, backup);
  try (Directory dir = new NIOFSDirectory(backup);
       IndexReader reader = DirectoryReader.open(dir)) {
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
    assertEquals(nDocs, hits.totalHits.value);
  }
}
 
Example #12
Source File: TestSolrCoreSnapshots.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private List<IndexCommit> listCommits(String directory) throws Exception {
  Directory dir = new NIOFSDirectory(Paths.get(directory));
  try {
    return DirectoryReader.listCommits(dir);
  } catch (IndexNotFoundException ex) {
    // This can happen when the delete snapshot functionality cleans up the index files (when the directory
    // storing these files is not the *current* index directory).
    return Collections.emptyList();
  }
}
 
Example #13
Source File: LogSearch.java    From exhibitor with Apache License 2.0 5 votes vote down vote up
public LogSearch(File file) throws Exception
{
    this.file = file;
    directory = new NIOFSDirectory(file, new NativeFSLockFactory());
    reader = IndexReader.open(directory);
    searcher = new IndexSearcher(reader);
}
 
Example #14
Source File: IndexStoreTests.java    From crate with Apache License 2.0 4 votes vote down vote up
private void doTestStoreDirectory(Index index,
                                  Path tempDir,
                                  String typeSettingValue,
                                  IndexModule.Type type) throws IOException {
    Settings.Builder settingsBuilder = Settings.builder()
        .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    if (typeSettingValue != null) {
        settingsBuilder.put(IndexModule.INDEX_STORE_TYPE_SETTING.getKey(), typeSettingValue);
    }
    Settings settings = settingsBuilder.build();
    IndexSettings indexSettings = IndexSettingsModule.newIndexSettings("foo", settings);
    FsDirectoryService service = new FsDirectoryService(
        indexSettings, new ShardPath(false, tempDir, tempDir, new ShardId(index, 0)));
    try (Directory directory = service.newFSDirectory(tempDir, NoLockFactory.INSTANCE)) {
        switch (type) {
            case HYBRIDFS:
                assertHybridDirectory(directory);
                break;
            case NIOFS:
                assertTrue(type + " " + directory.toString(), directory instanceof NIOFSDirectory);
                break;
            case MMAPFS:
                assertTrue(type + " " + directory.toString(), directory instanceof MMapDirectory);
                break;
            case SIMPLEFS:
                assertTrue(type + " " + directory.toString(), directory instanceof SimpleFSDirectory);
                break;
            case FS:
                if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
                    assertHybridDirectory(directory);
                } else if (Constants.WINDOWS) {
                    assertTrue(directory.toString(), directory instanceof SimpleFSDirectory);
                } else {
                    assertTrue(directory.toString(), directory instanceof NIOFSDirectory);
                }
                break;
            default:
                fail();
        }
    }
}
 
Example #15
Source File: StandardSearchEngine.java    From document-management-software with GNU Lesser General Public License v3.0 4 votes vote down vote up
static Directory getIndexDataDirectory() throws IOException {
	return new NIOFSDirectory(getIndexDataFolder().toPath());
}
 
Example #16
Source File: LuceneVsLuceneTest.java    From orientdb-lucene with Apache License 2.0 4 votes vote down vote up
protected Directory getDirectory() throws IOException {
  return NIOFSDirectory.open(getPath());
}
 
Example #17
Source File: LocalFileSystemRepository.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyFileTo(URI sourceDir, String fileName, Directory dest) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(sourceDir), NoLockFactory.INSTANCE)) {
    dest.copyFrom(dir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
  }
}
 
Example #18
Source File: LocalFileSystemRepository.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void copyFileFrom(Directory sourceDir, String fileName, URI dest) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dest), NoLockFactory.INSTANCE)) {
    dir.copyFrom(sourceDir, fileName, fileName, DirectoryFactory.IOCONTEXT_NO_CACHE);
  }
}
 
Example #19
Source File: LocalFileSystemRepository.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public String[] listAll(URI dirPath) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
    return dir.listAll();
  }
}
 
Example #20
Source File: LocalFileSystemRepository.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public IndexInput openInput(URI dirPath, String fileName, IOContext ctx) throws IOException {
  try (FSDirectory dir = new NIOFSDirectory(Paths.get(dirPath), NoLockFactory.INSTANCE)) {
    return dir.openInput(fileName, ctx);
  }
}
 
Example #21
Source File: NIOFSDirectoryFactory.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
  // we pass NoLockFactory, because the real lock factory is set later by injectLockFactory:
  return new NIOFSDirectory(new File(path).toPath(), lockFactory);
}
 
Example #22
Source File: TestBackwardsCompatibility.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testCommandLineArgs() throws Exception {

    PrintStream savedSystemOut = System.out;
    System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
    try {
      for (Map.Entry<String,Directory> entry : oldIndexDirs.entrySet()) {
        String name = entry.getKey();
        int indexCreatedVersion = SegmentInfos.readLatestCommit(entry.getValue()).getIndexCreatedVersionMajor();
        Path dir = createTempDir(name);
        TestUtil.unzip(getDataInputStream("index." + name + ".zip"), dir);
        
        String path = dir.toAbsolutePath().toString();
        
        List<String> args = new ArrayList<>();
        if (random().nextBoolean()) {
          args.add("-verbose");
        }
        if (random().nextBoolean()) {
          args.add("-delete-prior-commits");
        }
        if (random().nextBoolean()) {
          // TODO: need to better randomize this, but ...
          //  - LuceneTestCase.FS_DIRECTORIES is private
          //  - newFSDirectory returns BaseDirectoryWrapper
          //  - BaseDirectoryWrapper doesn't expose delegate
          Class<? extends FSDirectory> dirImpl = NIOFSDirectory.class;
          
          args.add("-dir-impl");
          args.add(dirImpl.getName());
        }
        args.add(path);
        
        IndexUpgrader upgrader = null;
        try {
          upgrader = IndexUpgrader.parseArgs(args.toArray(new String[0]));
        } catch (Exception e) {
          throw new AssertionError("unable to parse args: " + args, e);
        }
        upgrader.upgrade();
        
        Directory upgradedDir = newFSDirectory(dir);
        try {
          checkAllSegmentsUpgraded(upgradedDir, indexCreatedVersion);
        } finally {
          upgradedDir.close();
        }
      }
    } finally {
      System.setOut(savedSystemOut);
    }
  }
 
Example #23
Source File: TestIndexWriter.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testWithPendingDeletions() throws Exception {
  // irony: currently we don't emulate windows well enough to work on windows!
  assumeFalse("windows is not supported", Constants.WINDOWS);

  Path path = createTempDir();

  // Use WindowsFS to prevent open files from being deleted:
  FileSystem fs = new WindowsFS(path.getFileSystem()).getFileSystem(URI.create("file:///"));
  Path root = new FilterPath(path, fs);
  IndexCommit indexCommit;
  DirectoryReader reader;
  // MMapDirectory doesn't work because it closes its file handles after mapping!
  try (FSDirectory dir = new NIOFSDirectory(root)) {
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    IndexWriter w = new IndexWriter(dir, iwc);
    w.commit();
    reader = w.getReader();
    // we pull this commit to open it again later to check that we fail if a future file delete is pending
    indexCommit = reader.getIndexCommit();
    w.close();
    w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
    w.addDocument(new Document());
    w.close();
    IndexInput in = dir.openInput("segments_2", IOContext.DEFAULT);
    dir.deleteFile("segments_2");
    assertTrue(dir.getPendingDeletions().size() > 0);

    // make sure we get NoSuchFileException if we try to delete and already-pending-delete file:
    expectThrows(NoSuchFileException.class, () -> {
      dir.deleteFile("segments_2");
    });

    try (IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setIndexCommit(indexCommit))) {
      writer.addDocument(new Document());
      writer.commit();
      assertEquals(1, writer.getDocStats().maxDoc);
      // now check that we moved to 3
      dir.openInput("segments_3", IOContext.READ).close();;
    }
    reader.close();
    in.close();
  }
}