Java Code Examples for org.apache.lucene.index.IndexWriterConfig

The following examples show how to use org.apache.lucene.index.IndexWriterConfig. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public void initIndex() throws IOException {
	
	if(dir==null)
		open();
	
	IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
	  				   iwc.setOpenMode(OpenMode.CREATE);
    IndexWriter indexWriter = new IndexWriter(dir, iwc);
    
	for(MagicCard mc : MTGControler.getInstance().getEnabled(MTGCardsProvider.class).listAllCards())
	{
		try {
			indexWriter.addDocument(toDocuments(mc));
		} catch (IllegalArgumentException e) {
			logger.error("Error indexing " + mc + " " + mc.getCurrentSet(),e);
		}
	}
	
	indexWriter.commit();
	indexWriter.close();
}
 
Example 2
Source Project: lucene-solr   Source File: PayloadHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Sets up a RAM-resident Directory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new ByteBuffersDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = DirectoryReader.open(writer);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
  searcher.setSimilarity(similarity);
  return searcher;
}
 
Example 3
Source Project: lucene-solr   Source File: DocumentDictionaryTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 
Example 4
Source Project: Quelea   Source File: BibleSearchIndex.java    License: GNU General Public License v3.0 6 votes vote down vote up
/**
 * Add a number of chapters to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param bibleList the list of chapters to add.
 */
@Override
public void addAll(Collection<? extends BibleChapter> bibleList) {
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for(BibleChapter chapter : bibleList) {
            Document doc = new Document();
            doc.add(new TextField("text", chapter.getText(), Field.Store.NO));
            doc.add(new TextField("number", Integer.toString(chapter.getID()), Field.Store.YES));
            writer.addDocument(doc);
            chapters.put(chapter.getID(), chapter);
            LOGGER.log(Level.FINE, "Added bible chapter to index: {0}", chapter.getID());
        }
    }
    catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 
Example 5
Source Project: lumongo   Source File: BasicStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void cleanDatabaseAndInit() throws Exception {

	MongoClient mongo = TestHelper.getMongo();
	mongo.dropDatabase(TestHelper.TEST_DATABASE_NAME);
	directory = new DistributedDirectory(new MongoDirectory(mongo, TestHelper.TEST_DATABASE_NAME, STORAGE_TEST_INDEX, false));

	StandardAnalyzer analyzer = new StandardAnalyzer();
	IndexWriterConfig config = new IndexWriterConfig(analyzer);

	IndexWriter w = new IndexWriter(directory, config);

	addDoc(w, "Random perl Title that is long", "id-1");
	addDoc(w, "Random java Title that is long", "id-1");
	addDoc(w, "MongoDB is awesome", "id-2");
	addDoc(w, "This is a long title with nothing interesting", "id-3");
	addDoc(w, "Java is awesome", "id-4");
	addDoc(w, "Really big fish", "id-5");

	w.commit();
	w.close();
}
 
Example 6
Source Project: lucene-solr   Source File: TestPerFieldPostingsFormat2.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testMergeUnusedPerFieldCodec() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setOpenMode(OpenMode.CREATE).setCodec(new MockCodec());
  IndexWriter writer = newWriter(dir, iwconf);
  addDocs(writer, 10);
  writer.commit();
  addDocs3(writer, 10);
  writer.commit();
  addDocs2(writer, 10);
  writer.commit();
  assertEquals(30, writer.getDocStats().maxDoc);
  TestUtil.checkIndex(dir);
  writer.forceMerge(1);
  assertEquals(30, writer.getDocStats().maxDoc);
  writer.close();
  dir.close();
}
 
Example 7
Source Project: incubator-retired-blur   Source File: BlurUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example 8
Source Project: dacapobench   Source File: Index.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriterConfig IWConfig = new IndexWriterConfig();
  IWConfig.setOpenMode (IndexWriterConfig.OpenMode.CREATE);
  IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
  IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.forceMerge(1);
  }
  writer.close();
}
 
Example 9
Source Project: crate   Source File: GroupByOptimizedIteratorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testHighCardinalityRatioReturnsTrueForLowCardinality() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10; i++) {
        Document doc = new Document();
        BytesRef value = new BytesRef("1");
        doc.add(new Field(columnName, value, KeywordFieldMapper.Defaults.FIELD_TYPE.clone()));
        iw.addDocument(doc);
    }
    iw.commit();

    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    assertThat(
        GroupByOptimizedIterator.hasHighCardinalityRatio(() -> new Engine.Searcher("dummy", indexSearcher, () -> {}), "x"),
        is(false)
    );
}
 
Example 10
Source Project: geode-examples   Source File: SpatialHelperTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 
Example 11
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example 12
Source Project: RedisDirectory   Source File: TestLucene.java    License: Apache License 2.0 6 votes vote down vote up
public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}
 
Example 13
Source Project: lucene-solr   Source File: QueryAutoStopWordAnalyzerTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = new ByteBuffersDirectory();
  appAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(appAnalyzer));
  int numDocs = 200;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
    String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.length];
    doc.add(new TextField("variedField", variedFieldValue, Field.Store.YES));
    doc.add(new TextField("repetitiveField", repetitiveFieldValue, Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.close();
  reader = DirectoryReader.open(dir);
}
 
Example 14
Source Project: lucene-solr   Source File: TestSearcherManager.java    License: Apache License 2.0 6 votes vote down vote up
public void testEnsureOpen() throws Exception {
  Directory dir = newDirectory();
  new IndexWriter(dir, new IndexWriterConfig(null)).close();
  SearcherManager sm = new SearcherManager(dir, null);
  IndexSearcher s = sm.acquire();
  sm.close();
  
  // this should succeed;
  sm.release(s);
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.acquire();
  });
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.maybeRefresh();
  });

  dir.close();
}
 
Example 15
Source Project: lucene-solr   Source File: BaseLockFactoryTestCase.java    License: Apache License 2.0 6 votes vote down vote up
public void testStressLocks() throws Exception {
  Path tempPath = createTempDir();
  assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));

  Directory dir = getDirectory(tempPath);

  // First create a 1 doc index:
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
  addDoc(w);
  w.close();
  
  int numIterations = atLeast(20);
  WriterThread writer = new WriterThread(numIterations, dir);
  SearcherThread searcher = new SearcherThread(numIterations, dir);
  writer.start();
  searcher.start();

  writer.join();
  searcher.join();

  assertTrue("IndexWriter hit unexpected exceptions", !writer.hitException);
  assertTrue("IndexSearcher hit unexpected exceptions", !searcher.hitException);
  
  dir.close();
}
 
Example 16
Source Project: SnowGraph   Source File: CodePatternSearcher.java    License: Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example 17
Source Project: lucene-solr   Source File: RangeFacetsExample.java    License: Apache License 2.0 6 votes vote down vote up
/** Build the example index. */
public void index() throws IOException {
  IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
      new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));

  // Add documents with a fake timestamp, 1000 sec before
  // "now", 2000 sec before "now", ...:
  for(int i=0;i<100;i++) {
    Document doc = new Document();
    long then = nowSec - i * 1000;
    // Add as doc values field, so we can compute range facets:
    doc.add(new NumericDocValuesField("timestamp", then));
    // Add as numeric field so we can drill-down:
    doc.add(new LongPoint("timestamp", then));
    indexWriter.addDocument(doc);
  }

  // Open near-real-time searcher
  searcher = new IndexSearcher(DirectoryReader.open(indexWriter));
  indexWriter.close();
}
 
Example 18
Source Project: lucene-solr   Source File: CreateIndexTask.java    License: Apache License 2.0 6 votes vote down vote up
public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws IOException {
  IndexWriterConfig iwc = createWriterConfig(config, runData, mode, commit);
  String infoStreamVal = config.get("writer.info.stream", null);
  if (infoStreamVal != null) {
    if (infoStreamVal.equals("SystemOut")) {
      iwc.setInfoStream(System.out);
    } else if (infoStreamVal.equals("SystemErr")) {
      iwc.setInfoStream(System.err);
    } else {
      Path f = Paths.get(infoStreamVal);
      iwc.setInfoStream(new PrintStream(new BufferedOutputStream(Files.newOutputStream(f)), false, Charset.defaultCharset().name()));
    }
  }
  IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc);
  return writer;
}
 
Example 19
@Before
public void setupIndex() throws Exception {
    dir = new ByteBuffersDirectory();

    try(IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
        for (int i = 0; i < docs.length; i++) {
            Document doc = new Document();
            doc.add(new Field("_id", Integer.toString(i + 1), StoredField.TYPE));
            doc.add(newTextField("text", docs[i], Field.Store.YES));
            indexWriter.addDocument(doc);
        }
    }

    reader = DirectoryReader.open(dir);
    searcher = new IndexSearcher(reader);
}
 
Example 20
@Test
public void testNativeLockErrorOnStartup() throws Exception {

  File indexDir = new File(initAndGetDataDir(), "index");
  if (log.isInfoEnabled()) {
    log.info("Acquiring lock on {}", indexDir.getAbsolutePath());
  }
  Directory directory = newFSDirectory(indexDir.toPath(), NativeFSLockFactory.INSTANCE);
  //creates a new IndexWriter without releasing the lock yet
  IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig(null));

  ignoreException("locked");
  try {
    System.setProperty("solr.tests.lockType",DirectoryFactory.LOCK_TYPE_NATIVE);
    //opening a new core on the same index
    initCore("solrconfig-basic.xml", "schema.xml");
    CoreContainer cc = h.getCoreContainer();
    if (checkForCoreInitException(LockObtainFailedException.class))
      return;
    fail("Expected " + LockObtainFailedException.class.getSimpleName());
  } finally {
    System.clearProperty("solr.tests.lockType");
    unIgnoreException("locked");
    indexWriter.close();
    directory.close();
    deleteCore();
  }
}
 
Example 21
Source Project: vscode-extension   Source File: test.java    License: MIT License 5 votes vote down vote up
IndexWriter createWriter(Directory directory, IndexWriterConfig iwc) throws IOException {
    if (Assertions.ENABLED) {
        return new AssertingIndexWriter(directory, iwc);
    } else {
        return new IndexWriter(directory, iwc);
    }
}
 
Example 22
Source Project: lucene-solr   Source File: TestFeatureSort.java    License: Apache License 2.0 5 votes vote down vote up
public void testFeature() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "name", 30.1F));
  doc.add(newStringField("value", "30.1", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1.3F));
  doc.add(newStringField("value", "1.3", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4.2F));
  doc.add(newStringField("value", "4.2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(FeatureField.newFeatureSort("field", "name"));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits.value);
  // numeric order
  assertEquals("30.1", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("4.2", searcher.doc(td.scoreDocs[1].doc).get("value"));
  assertEquals("1.3", searcher.doc(td.scoreDocs[2].doc).get("value"));

  ir.close();
  dir.close();
}
 
Example 23
Source Project: incubator-retired-blur   Source File: CoreTestContext.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Index will contain 26 documents with the following column/values: alpha =
 * double-letter a-z (lowercase characters); num = 0-25 val = val (constant
 * across all docs)
 * 
 * New columns may be added so don't rely on the column count in tests.
 * 
 * @return
 */
public static IndexContext newSimpleAlpaNumContext() {
  CoreTestContext ctx = new CoreTestContext();

  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
  try {
    IndexWriter writer = new IndexWriter(ctx.directory, conf);

    for (int i = 0; i < 26; i++) {
      String alpha = new Character((char) (97 + i)).toString();
      Document doc = new Document();

      doc.add(new Field("id", Integer.toString(i), TextField.TYPE_STORED));
      doc.add(new Field("alpha", alpha + alpha, TextField.TYPE_STORED));
      doc.add(new Field("num", Integer.toString(i), TextField.TYPE_STORED));
      doc.add(new Field("val", "val", TextField.TYPE_STORED));

      writer.addDocument(doc);

      writer.commit();
    }
    writer.commit();
    writer.close();
  } catch (IOException e) {
    throw new RuntimeException("Unable to create test context.", e);
  }

  return ctx;
}
 
Example 24
Source Project: Quelea   Source File: SearchIndexUtils.java    License: GNU General Public License v3.0 5 votes vote down vote up
/**
 * Clear the given index.
 * <p/>
 * @param index the index to clear.
 */
public static void clearIndex(Directory index) {
    try(IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(new StandardAnalyzer(CharArraySet.EMPTY_SET)))) {
        writer.deleteAll();
        writer.commit();
    }
    catch(IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't clear the index", ex);
    }
}
 
Example 25
Source Project: taoshop   Source File: LuceneIndexer.java    License: Apache License 2.0 5 votes vote down vote up
public boolean createIndex(String indexDir) throws IOException{
    //加点测试的静态数据
    Integer ids[] = {1 , 2 , 3};
    String titles[] = {"标题1" , "标题2" , "标题3"};
    String tcontents[] = {
            "内容1内容啊哈哈哈",
            "内容2内容啊哈哈哈",
            "内容3内容啊哈哈哈"
    };

    long startTime = System.currentTimeMillis();//记录索引开始时间

    Analyzer analyzer = new SmartChineseAnalyzer();
    Directory directory = FSDirectory.open(Paths.get(indexDir));
    IndexWriterConfig config = new IndexWriterConfig(analyzer);

    IndexWriter indexWriter = new IndexWriter(directory, config);

    for(int i = 0; i < ids.length;i++){
        Document doc = new Document();
        //添加字段
        doc.add(new TextField("id", ids[i].toString(),Field.Store.YES)); //添加内容
        doc.add(new TextField("title", titles[i], Field.Store.YES)); //添加文件名,并把这个字段存到索引文件里
        doc.add(new TextField("tcontent", tcontents[i], Field.Store.YES)); //添加文件路径
        indexWriter.addDocument(doc);
    }

    indexWriter.commit();
    System.out.println("共索引了"+indexWriter.numDocs()+"个文件");
    indexWriter.close();
    System.out.println("创建索引所用时间:"+(System.currentTimeMillis()-startTime)+"毫秒");

    return true;
}
 
Example 26
Source Project: taoshop   Source File: LuceneIndexer.java    License: Apache License 2.0 5 votes vote down vote up
public boolean createIndex(String indexDir) throws IOException{
	//加点测试的静态数据
	Integer ids[] = {1 , 2 , 3};
	String titles[] = {"标题1" , "标题2" , "标题3"};
	String tcontents[] = {
			"内容1内容啊哈哈哈",
			"内容2内容啊哈哈哈",
			"内容3内容啊哈哈哈"
	};

	long startTime = System.currentTimeMillis();//记录索引开始时间
	
	Analyzer analyzer = new SmartChineseAnalyzer();
	Directory directory = FSDirectory.open(Paths.get(indexDir));
	IndexWriterConfig config = new IndexWriterConfig(analyzer);
	
	IndexWriter indexWriter = new IndexWriter(directory, config);

	for(int i = 0; i < ids.length;i++){
		Document doc = new Document();
		//添加字段
        doc.add(new TextField("id", ids[i].toString(),Field.Store.YES)); //添加内容
        doc.add(new TextField("title", titles[i], Field.Store.YES)); //添加文件名,并把这个字段存到索引文件里
        doc.add(new TextField("tcontent", tcontents[i], Field.Store.YES)); //添加文件路径
        indexWriter.addDocument(doc);
	}

	indexWriter.commit();
	System.out.println("共索引了"+indexWriter.numDocs()+"个文件");
	indexWriter.close();
	System.out.println("创建索引所用时间:"+(System.currentTimeMillis()-startTime)+"毫秒");
	
	return true;
}
 
Example 27
Source Project: lucene-solr   Source File: IndexRevisionTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpen() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(null);
  conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
  IndexWriter writer = new IndexWriter(dir, conf);
  try {
    writer.addDocument(new Document());
    writer.commit();
    Revision rev = new IndexRevision(writer);
    @SuppressWarnings("unchecked")
    Map<String, List<RevisionFile>> sourceFiles = rev.getSourceFiles();
    String source = sourceFiles.keySet().iterator().next();
    for (RevisionFile file : sourceFiles.values().iterator().next()) {
      IndexInput src = dir.openInput(file.fileName, IOContext.READONCE);
      InputStream in = rev.open(source, file.fileName);
      assertEquals(src.length(), in.available());
      byte[] srcBytes = new byte[(int) src.length()];
      byte[] inBytes = new byte[(int) src.length()];
      int offset = 0;
      if (random().nextBoolean()) {
        int skip = random().nextInt(10);
        if (skip >= src.length()) {
          skip = 0;
        }
        in.skip(skip);
        src.seek(skip);
        offset = skip;
      }
      src.readBytes(srcBytes, offset, srcBytes.length - offset);
      in.read(inBytes, offset, inBytes.length - offset);
      assertArrayEquals(srcBytes, inBytes);
      IOUtils.close(src, in);
    }
    writer.close();
  } finally {
    IOUtils.close(dir);
  }
}
 
Example 28
Source Project: lucene-solr   Source File: TestFeatureSort.java    License: Apache License 2.0 5 votes vote down vote up
public void testFeatureMissingFeatureNameInSegment() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, config);
  Document doc = new Document();
  doc.add(new FeatureField("field", "different_name", 0.5F));
  writer.addDocument(doc);
  writer.commit();
  doc = new Document();
  doc.add(new FeatureField("field", "name", 1.3F));
  doc.add(newStringField("value", "1.3", Field.Store.YES));
  writer.addDocument(doc);
  doc = new Document();
  doc.add(new FeatureField("field", "name", 4.2F));
  doc.add(newStringField("value", "4.2", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(FeatureField.newFeatureSort("field", "name"));

  TopDocs td = searcher.search(new MatchAllDocsQuery(), 10, sort);
  assertEquals(3, td.totalHits.value);
  // null is treated as 0
  assertEquals("4.2", searcher.doc(td.scoreDocs[0].doc).get("value"));
  assertEquals("1.3", searcher.doc(td.scoreDocs[1].doc).get("value"));
  assertNull(searcher.doc(td.scoreDocs[2].doc).get("value"));

  ir.close();
  dir.close();
}
 
Example 29
Source Project: gerbil   Source File: Indexer.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
public static Indexer create(String indexDirPath) {
    Directory indexDirectory = null;
    try {
        indexDirectory = FSDirectory.open(new File(indexDirPath).toPath());
        IndexWriterConfig config = new IndexWriterConfig();
        config.setOpenMode(OpenMode.CREATE);
        IndexWriter indexWriter = new IndexWriter(indexDirectory, config);
        return new Indexer(indexDirectory, indexWriter);
    } catch (IOException e) {
        LOGGER.error("Exception while trying to create index writer for entity checking. Returning null.", e);
        IOUtils.closeQuietly(indexDirectory);
        return null;
    }
}
 
Example 30
Source Project: jstarcraft-core   Source File: LuceneEngine.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 合并管理器
 * 
 * @throws Exception
 */
void mergeManager() throws Exception {
    writeLock.lock();
    TransienceManager newTransienceManager = new TransienceManager((IndexWriterConfig) BeanUtils.cloneBean(config), new ByteBuffersDirectory());
    TransienceManager oldTransienceManager = this.transienceManager;
    try {
        lockWrite();
        this.transienceManager = newTransienceManager;
        // 触发变更
        this.persistenceManager.setManager(oldTransienceManager);
    } finally {
        unlockWrite();
    }

    // 此处需要防止有线程在使用时关闭.
    try {
        lockRead();
        // 只关闭writer,不关闭reader.
        oldTransienceManager.close();
    } finally {
        unlockRead();
    }

    this.persistenceManager.mergeManager();

    try {
        lockWrite();
        // 触发变更
        this.persistenceManager.setManager(null);
    } finally {
        unlockWrite();
    }
    writeLock.unlock();
}