Java Code Examples for org.apache.lucene.index.IndexWriterConfig

The following examples show how to use org.apache.lucene.index.IndexWriterConfig. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: MtgDesktopCompanion   Author: nicho92   File: LuceneIndexer.java    License: GNU General Public License v3.0 6 votes vote down vote up
public void initIndex() throws IOException {
	
	if(dir==null)
		open();
	
	IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
	  				   iwc.setOpenMode(OpenMode.CREATE);
    IndexWriter indexWriter = new IndexWriter(dir, iwc);
    
	for(MagicCard mc : MTGControler.getInstance().getEnabled(MTGCardsProvider.class).listAllCards())
	{
		try {
			indexWriter.addDocument(toDocuments(mc));
		} catch (IllegalArgumentException e) {
			logger.error("Error indexing " + mc + " " + mc.getCurrentSet(),e);
		}
	}
	
	indexWriter.commit();
	indexWriter.close();
}
 
Example #2
Source Project: lucene-solr   Author: apache   File: PayloadHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets up a RAM-resident Directory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new ByteBuffersDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = DirectoryReader.open(writer);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
  searcher.setSimilarity(similarity);
  return searcher;
}
 
Example #3
Source Project: lucene-solr   Author: apache   File: DocumentDictionaryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 
Example #4
Source Project: Quelea   Author: quelea-projection   File: BibleSearchIndex.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Add a number of chapters to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param bibleList the list of chapters to add.
 */
@Override
public void addAll(Collection<? extends BibleChapter> bibleList) {
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for(BibleChapter chapter : bibleList) {
            Document doc = new Document();
            doc.add(new TextField("text", chapter.getText(), Field.Store.NO));
            doc.add(new TextField("number", Integer.toString(chapter.getID()), Field.Store.YES));
            writer.addDocument(doc);
            chapters.put(chapter.getID(), chapter);
            LOGGER.log(Level.FINE, "Added bible chapter to index: {0}", chapter.getID());
        }
    }
    catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 
Example #5
Source Project: lumongo   Author: lumongo   File: BasicStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void cleanDatabaseAndInit() throws Exception {

	MongoClient mongo = TestHelper.getMongo();
	mongo.dropDatabase(TestHelper.TEST_DATABASE_NAME);
	directory = new DistributedDirectory(new MongoDirectory(mongo, TestHelper.TEST_DATABASE_NAME, STORAGE_TEST_INDEX, false));

	StandardAnalyzer analyzer = new StandardAnalyzer();
	IndexWriterConfig config = new IndexWriterConfig(analyzer);

	IndexWriter w = new IndexWriter(directory, config);

	addDoc(w, "Random perl Title that is long", "id-1");
	addDoc(w, "Random java Title that is long", "id-1");
	addDoc(w, "MongoDB is awesome", "id-2");
	addDoc(w, "This is a long title with nothing interesting", "id-3");
	addDoc(w, "Java is awesome", "id-4");
	addDoc(w, "Really big fish", "id-5");

	w.commit();
	w.close();
}
 
Example #6
Source Project: lucene-solr   Author: apache   File: TestPerFieldPostingsFormat2.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMergeUnusedPerFieldCodec() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setOpenMode(OpenMode.CREATE).setCodec(new MockCodec());
  IndexWriter writer = newWriter(dir, iwconf);
  addDocs(writer, 10);
  writer.commit();
  addDocs3(writer, 10);
  writer.commit();
  addDocs2(writer, 10);
  writer.commit();
  assertEquals(30, writer.getDocStats().maxDoc);
  TestUtil.checkIndex(dir);
  writer.forceMerge(1);
  assertEquals(30, writer.getDocStats().maxDoc);
  writer.close();
  dir.close();
}
 
Example #7
Source Project: incubator-retired-blur   Author: apache   File: BlurUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #8
Source Project: dacapobench   Author: dacapobench   File: Index.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriterConfig IWConfig = new IndexWriterConfig();
  IWConfig.setOpenMode (IndexWriterConfig.OpenMode.CREATE);
  IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
  IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.forceMerge(1);
  }
  writer.close();
}
 
Example #9
Source Project: crate   Author: crate   File: GroupByOptimizedIteratorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHighCardinalityRatioReturnsTrueForLowCardinality() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10; i++) {
        Document doc = new Document();
        BytesRef value = new BytesRef("1");
        doc.add(new Field(columnName, value, KeywordFieldMapper.Defaults.FIELD_TYPE.clone()));
        iw.addDocument(doc);
    }
    iw.commit();

    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    assertThat(
        GroupByOptimizedIterator.hasHighCardinalityRatio(() -> new Engine.Searcher("dummy", indexSearcher, () -> {}), "x"),
        is(false)
    );
}
 
Example #10
Source Project: geode-examples   Author: apache   File: SpatialHelperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 
Example #11
Source Project: incubator-retired-blur   Author: apache   File: FastHdfsKeyValueDirectoryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example #12
Source Project: RedisDirectory   Author: shijiebei2009   File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}
 
Example #13
Source Project: lucene-solr   Author: apache   File: QueryAutoStopWordAnalyzerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = new ByteBuffersDirectory();
  appAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(appAnalyzer));
  int numDocs = 200;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
    String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.length];
    doc.add(new TextField("variedField", variedFieldValue, Field.Store.YES));
    doc.add(new TextField("repetitiveField", repetitiveFieldValue, Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.close();
  reader = DirectoryReader.open(dir);
}
 
Example #14
Source Project: lucene-solr   Author: apache   File: TestSearcherManager.java    License: Apache License 2.0 6 votes vote down vote up
public void testEnsureOpen() throws Exception {
  Directory dir = newDirectory();
  new IndexWriter(dir, new IndexWriterConfig(null)).close();
  SearcherManager sm = new SearcherManager(dir, null);
  IndexSearcher s = sm.acquire();
  sm.close();
  
  // this should succeed;
  sm.release(s);
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.acquire();
  });
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.maybeRefresh();
  });

  dir.close();
}
 
Example #15
Source Project: lucene-solr   Author: apache   File: BaseLockFactoryTestCase.java    License: Apache License 2.0 6 votes vote down vote up
public void testStressLocks() throws Exception {
  Path tempPath = createTempDir();
  assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));

  Directory dir = getDirectory(tempPath);

  // First create a 1 doc index:
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
  addDoc(w);
  w.close();
  
  int numIterations = atLeast(20);
  WriterThread writer = new WriterThread(numIterations, dir);
  SearcherThread searcher = new SearcherThread(numIterations, dir);
  writer.start();
  searcher.start();

  writer.join();
  searcher.join();

  assertTrue("IndexWriter hit unexpected exceptions", !writer.hitException);
  assertTrue("IndexSearcher hit unexpected exceptions", !searcher.hitException);
  
  dir.close();
}
 
Example #16
Source Project: SnowGraph   Author: linzeqipku   File: CodePatternSearcher.java    License: Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example #17
Source Project: lucene-solr   Author: apache   File: RangeFacetsExample.java    License: Apache License 2.0 6 votes vote down vote up
/** Build the example index. */
public void index() throws IOException {
  IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
      new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));

  // Add documents with a fake timestamp, 1000 sec before
  // "now", 2000 sec before "now", ...:
  for(int i=0;i<100;i++) {
    Document doc = new Document();
    long then = nowSec - i * 1000;
    // Add as doc values field, so we can compute range facets:
    doc.add(new NumericDocValuesField("timestamp", then));
    // Add as numeric field so we can drill-down:
    doc.add(new LongPoint("timestamp", then));
    indexWriter.addDocument(doc);
  }

  // Open near-real-time searcher
  searcher = new IndexSearcher(DirectoryReader.open(indexWriter));
  indexWriter.close();
}
 
Example #18
Source Project: lucene-solr   Author: apache   File: CreateIndexTask.java    License: Apache License 2.0 6 votes vote down vote up
public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws IOException {
  IndexWriterConfig iwc = createWriterConfig(config, runData, mode, commit);
  String infoStreamVal = config.get("writer.info.stream", null);
  if (infoStreamVal != null) {
    if (infoStreamVal.equals("SystemOut")) {
      iwc.setInfoStream(System.out);
    } else if (infoStreamVal.equals("SystemErr")) {
      iwc.setInfoStream(System.err);
    } else {
      Path f = Paths.get(infoStreamVal);
      iwc.setInfoStream(new PrintStream(new BufferedOutputStream(Files.newOutputStream(f)), false, Charset.defaultCharset().name()));
    }
  }
  IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc);
  return writer;
}
 
Example #19
Source Project: elasticsearch-learning-to-rank   Author: o19s   File: ExplorerQueryTests.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setupIndex() throws Exception {
    dir = new ByteBuffersDirectory();

    try(IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
        for (int i = 0; i < docs.length; i++) {
            Document doc = new Document();
            doc.add(new Field("_id", Integer.toString(i + 1), StoredField.TYPE));
            doc.add(newTextField("text", docs[i], Field.Store.YES));
            indexWriter.addDocument(doc);
        }
    }

    reader = DirectoryReader.open(dir);
    searcher = new IndexSearcher(reader);
}
 
Example #20
Source Project: lucene-solr   Author: apache   File: SolrCoreCheckLockOnStartupTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testNativeLockErrorOnStartup() throws Exception {

  File indexDir = new File(initAndGetDataDir(), "index");
  if (log.isInfoEnabled()) {
    log.info("Acquiring lock on {}", indexDir.getAbsolutePath());
  }
  Directory directory = newFSDirectory(indexDir.toPath(), NativeFSLockFactory.INSTANCE);
  //creates a new IndexWriter without releasing the lock yet
  IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null));

  ignoreException("locked");
  try {
    System.setProperty("solr.tests.lockType",DirectoryFactory.LOCK_TYPE_NATIVE);
    //opening a new core on the same index
    initCore("solrconfig-basic.xml", "schema.xml");
    CoreContainer cc = h.getCoreContainer();
    if (checkForCoreInitException(LockObtainFailedException.class))
      return;
    fail("Expected " + LockObtainFailedException.class.getSimpleName());
  } finally {
    System.clearProperty("solr.tests.lockType");
    unIgnoreException("locked");
    indexWriter.close();
    directory.close();
    deleteCore();
  }
}
 
Example #21
Source Project: vscode-extension   Author: DeepCodeAI   File: test.java    License: MIT License 5 votes vote down vote up
IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
    if (Assertions.ENABLED) {
        return new AssertingIndexWriter(directory, iwc);
    } else {
        return new IndexWriter(directory, iwc);
    }
}
 
Example #22
Source Project: lucene-solr   Author: apache   File: TestFeatureSort.java    License: Apache License 2.0 5 votes vote down vote up
public void testFeature() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "name", 30.1F));
  doc.add(newStringField("value", "30.1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1.3F));
  doc.add(newStringField("value", "1.3", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4.2F));
  doc.add(newStringField("value", "4.2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(FeatureField.newFeatureSort("field", "name"));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits.value);
  // numeric order
  assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value"));
  assertEquals("1.3", searcher.doc(td.scoreDocs[2].doc).get("value"));

  ir.close();
  dir.close();
}
 
Example #23
Source Project: incubator-retired-blur   Author: apache   File: CoreTestContext.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Index will contain 26 documents with the following column/values: alpha =
 * double-letter a-z (lowercase characters); num = 0-25 val = val (constant
 * across all docs)
 * 
 * New columns may be added so don't rely on the column count in tests.
 * 
 * @return
 */
public static IndexContext newSimpleAlpaNumContext() {
  CoreTestContext ctx = new CoreTestContext();

  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
  try {
    IndexWriter writer = new IndexWriter(ctx.directory, conf);

    for (int i = 0; i < 26; i++) {
      String alpha = new Character((char) (97 + i)).toString();
      Document doc = new Document();

      doc.add(new Field("id", Integer.toString(i), TextField.TYPE_STORED));
      doc.add(new Field("alpha", alpha + alpha, TextField.TYPE_STORED));
      doc.add(new Field("num", Integer.toString(i), TextField.TYPE_STORED));
      doc.add(new Field("val", "val", TextField.TYPE_STORED));

      writer.addDocument(doc);

      writer.commit();
    }
    writer.commit();
    writer.close();
  } catch (IOException e) {
    throw new RuntimeException("Unable to create test context.", e);
  }

  return ctx;
}
 
Example #24
Source Project: Quelea   Author: quelea-projection   File: SearchIndexUtils.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * Clear the given index.
 * <p/>
 * @param index the index to clear.
 */
public static void clearIndex(Directory index) {
    try(IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer(CharArraySet.EMPTY_SET)))) {
        writer.deleteAll();
        writer.commit();
    }
    catch(IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't clear the index", ex);
    }
}
 
Example #25
Source Project: taoshop   Author: u014427391   File: LuceneIndexer.java    License: Apache License 2.0 5 votes vote down vote up
public boolean createIndex(String indexDir) throws IOException{
    //加点测试的静态数据
    Integer ids[] = {1 , 2 , 3};
    String titles[] = {"标题1" , "标题2" , "标题3"};
    String tcontents[] = {
            "内容1内容啊哈哈哈",
            "内容2内容啊哈哈哈",
            "内容3内容啊哈哈哈"
    };

    long startTime = System.currentTimeMillis();//记录索引开始时间

    Analyzer analyzer = new SmartChineseAnalyzer();
    Directory directory = FSDirectory.open(Paths.get(indexDir));
    IndexWriterConfig config = new IndexWriterConfig(analyzer);

    IndexWriter indexWriter = new IndexWriter(directory, config);

    for(int i = 0; i < ids.length;i++){
        Document doc = new Document();
        //添加字段
        doc.add(new TextField("id", ids[i].toString(),Field.Store.YES)); //添加内容
        doc.add(new TextField("title", titles[i], Field.Store.YES)); //添加文件名,并把这个字段存到索引文件里
        doc.add(new TextField("tcontent", tcontents[i], Field.Store.YES)); //添加文件路径
        indexWriter.addDocument(doc);
    }

    indexWriter.commit();
    System.out.println("共索引了"+indexWriter.numDocs()+"个文件");
    indexWriter.close();
    System.out.println("创建索引所用时间:"+(System.currentTimeMillis()-startTime)+"毫秒");

    return true;
}
 
Example #26
Source Project: taoshop   Author: u014427391   File: LuceneIndexer.java    License: Apache License 2.0 5 votes vote down vote up
public boolean createIndex(String indexDir) throws IOException{
	//加点测试的静态数据
	Integer ids[] = {1 , 2 , 3};
	String titles[] = {"标题1" , "标题2" , "标题3"};
	String tcontents[] = {
			"内容1内容啊哈哈哈",
			"内容2内容啊哈哈哈",
			"内容3内容啊哈哈哈"
	};

	long startTime = System.currentTimeMillis();//记录索引开始时间
	
	Analyzer analyzer = new SmartChineseAnalyzer();
	Directory directory = FSDirectory.open(Paths.get(indexDir));
	IndexWriterConfig config = new IndexWriterConfig(analyzer);
	
	IndexWriter indexWriter = new IndexWriter(directory, config);

	for(int i = 0; i < ids.length;i++){
		Document doc = new Document();
		//添加字段
        doc.add(new TextField("id", ids[i].toString(),Field.Store.YES)); //添加内容
        doc.add(new TextField("title", titles[i], Field.Store.YES)); //添加文件名,并把这个字段存到索引文件里
        doc.add(new TextField("tcontent", tcontents[i], Field.Store.YES)); //添加文件路径
        indexWriter.addDocument(doc);
	}

	indexWriter.commit();
	System.out.println("共索引了"+indexWriter.numDocs()+"个文件");
	indexWriter.close();
	System.out.println("创建索引所用时间:"+(System.currentTimeMillis()-startTime)+"毫秒");
	
	return true;
}
 
Example #27
Source Project: lucene-solr   Author: apache   File: IndexRevisionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpen() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    String source = sourceFiles.keySet().iterator().next();
    for (RevisionFile file : sourceFiles.values().iterator().next()) {
      IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
      InputStream in = rev.open(source, file.fileName);
      assertEquals(src.length(), in.available());
      byte[] srcBytes = new byte[(int) src.length()];
      byte[] inBytes = new byte[(int) src.length()];
      int offset = 0;
      if (random().nextBoolean()) {
        int skip = random().nextInt(10);
        if (skip >= src.length()) {
          skip = 0;
        }
        in.skip(skip);
        src.seek(skip);
        offset = skip;
      }
      src.readBytes(srcBytes, offset, srcBytes.length - offset);
      in.read(inBytes, offset, inBytes.length - offset);
      assertArrayEquals(srcBytes, inBytes);
      IOUtils.close(src, in);
    }
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 
Example #28
Source Project: lucene-solr   Author: apache   File: TestFeatureSort.java    License: Apache License 2.0 5 votes vote down vote up
public void testFeatureMissingFeatureNameInSegment() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "different_name", 0.5F));
  writer.addDocument(doc);
  writer.commit();
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1.3F));
  doc.add(newStringField("value", "1.3", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4.2F));
  doc.add(newStringField("value", "4.2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(FeatureField.newFeatureSort("field", "name"));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits.value);
  // null is treated as 0
  assertEquals("4.2", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("1.3", searcher.doc(td.scoreDocs[1].doc).get("value"));
  assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));

  ir.close();
  dir.close();
}
 
Example #29
Source Project: gerbil   Author: dice-group   File: Indexer.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
public static Indexer create(String indexDirPath) {
    Directory indexDirectory = null;
    try {
        indexDirectory = FSDirectory.open(new File(indexDirPath).toPath());
        IndexWriterConfig config = new IndexWriterConfig();
        config.setOpenMode(OpenMode.CREATE);
        IndexWriter indexWriter = new IndexWriter(indexDirectory, config);
        return new Indexer(indexDirectory, indexWriter);
    } catch (IOException e) {
        LOGGER.error("Exception while trying to create index writer for entity checking. Returning null.", e);
        IOUtils.closeQuietly(indexDirectory);
        return null;
    }
}
 
Example #30
Source Project: jstarcraft-core   Author: HongZhaoHua   File: LuceneEngine.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 合并管理器
 * 
 * @throws Exception
 */
void mergeManager() throws Exception {
    writeLock.lock();
    TransienceManager newTransienceManager = new TransienceManager((IndexWriterConfig) BeanUtils.cloneBean(config), new ByteBuffersDirectory());
    TransienceManager oldTransienceManager = this.transienceManager;
    try {
        lockWrite();
        this.transienceManager = newTransienceManager;
        // 触发变更
        this.persistenceManager.setManager(oldTransienceManager);
    } finally {
        unlockWrite();
    }

    // 此处需要防止有线程在使用时关闭.
    try {
        lockRead();
        // 只关闭writer,不关闭reader.
        oldTransienceManager.close();
    } finally {
        unlockRead();
    }

    this.persistenceManager.mergeManager();

    try {
        lockWrite();
        // 触发变更
        this.persistenceManager.setManager(null);
    } finally {
        unlockWrite();
    }
    writeLock.unlock();
}