org.apache.lucene.index.IndexWriterConfig Java Examples

The following examples show how to use org.apache.lucene.index.IndexWriterConfig. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlurUtilsTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private IndexReader getReaderWithDocsHavingFamily() throws CorruptIndexException, LockObtainFailedException,
    IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f2", Store.YES));

  Document doc1 = new Document();
  doc1.add(new StringField("a", "b", Store.YES));
  doc1.add(new StringField("family", "f1", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example #2
Source File: BasicStorageTest.java    From lumongo with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void cleanDatabaseAndInit() throws Exception {

	MongoClient mongo = TestHelper.getMongo();
	mongo.dropDatabase(TestHelper.TEST_DATABASE_NAME);
	directory = new DistributedDirectory(new MongoDirectory(mongo, TestHelper.TEST_DATABASE_NAME, STORAGE_TEST_INDEX, false));

	StandardAnalyzer analyzer = new StandardAnalyzer();
	IndexWriterConfig config = new IndexWriterConfig(analyzer);

	IndexWriter w = new IndexWriter(directory, config);

	addDoc(w, "Random perl Title that is long", "id-1");
	addDoc(w, "Random java Title that is long", "id-1");
	addDoc(w, "MongoDB is awesome", "id-2");
	addDoc(w, "This is a long title with nothing interesting", "id-3");
	addDoc(w, "Java is awesome", "id-4");
	addDoc(w, "Really big fish", "id-5");

	w.commit();
	w.close();
}
 
Example #3
Source File: CreateIndexTask.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static IndexWriter configureWriter(Config config, PerfRunData runData, OpenMode mode, IndexCommit commit) throws IOException {
  IndexWriterConfig iwc = createWriterConfig(config, runData, mode, commit);
  String infoStreamVal = config.get("writer.info.stream", null);
  if (infoStreamVal != null) {
    if (infoStreamVal.equals("SystemOut")) {
      iwc.setInfoStream(System.out);
    } else if (infoStreamVal.equals("SystemErr")) {
      iwc.setInfoStream(System.err);
    } else {
      Path f = Paths.get(infoStreamVal);
      iwc.setInfoStream(new PrintStream(new BufferedOutputStream(Files.newOutputStream(f)), false, Charset.defaultCharset().name()));
    }
  }
  IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc);
  return writer;
}
 
Example #4
Source File: ExplorerQueryTests.java    From elasticsearch-learning-to-rank with Apache License 2.0 6 votes vote down vote up
@Before
public void setupIndex() throws Exception {
    dir = new ByteBuffersDirectory();

    try(IndexWriter indexWriter = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER))) {
        for (int i = 0; i < docs.length; i++) {
            Document doc = new Document();
            doc.add(new Field("_id", Integer.toString(i + 1), StoredField.TYPE));
            doc.add(newTextField("text", docs[i], Field.Store.YES));
            indexWriter.addDocument(doc);
        }
    }

    reader = DirectoryReader.open(dir);
    searcher = new IndexSearcher(reader);
}
 
Example #5
Source File: BibleSearchIndex.java    From Quelea with GNU General Public License v3.0 6 votes vote down vote up
/**
 * Add a number of chapters to the index. This is much more efficient than
 * calling add() repeatedly because it just uses one writer rather than
 * opening and closing one for each individual operation.
 *
 * @param bibleList the list of chapters to add.
 */
@Override
public void addAll(Collection<? extends BibleChapter> bibleList) {
    try (IndexWriter writer = new IndexWriter(index, new IndexWriterConfig(analyzer))) {
        for(BibleChapter chapter : bibleList) {
            Document doc = new Document();
            doc.add(new TextField("text", chapter.getText(), Field.Store.NO));
            doc.add(new TextField("number", Integer.toString(chapter.getID()), Field.Store.YES));
            writer.addDocument(doc);
            chapters.put(chapter.getID(), chapter);
            LOGGER.log(Level.FINE, "Added bible chapter to index: {0}", chapter.getID());
        }
    }
    catch (IOException ex) {
        LOGGER.log(Level.SEVERE, "Couldn't add value to index", ex);
    }
}
 
Example #6
Source File: DocumentDictionaryTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testEmptyReader() throws IOException {
  Directory dir = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  // Make sure the index is created?
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
  InputIterator inputIterator = dictionary.getEntryIterator();

  assertNull(inputIterator.next());
  assertEquals(inputIterator.weight(), 0);
  assertNull(inputIterator.payload());
  
  IOUtils.close(ir, analyzer, dir);
}
 
Example #7
Source File: PayloadHelper.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Sets up a RAM-resident Directory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
 * and analyzes them using the PayloadAnalyzer
 * @param similarity The Similarity class to use in the Searcher
 * @param numDocs The num docs to add
 * @return An IndexSearcher
 */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
  Directory directory = new MockDirectoryWrapper(random, new ByteBuffersDirectory());
  PayloadAnalyzer analyzer = new PayloadAnalyzer();

  // TODO randomize this
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(
      analyzer).setSimilarity(similarity));
  // writer.infoStream = System.out;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
    doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = DirectoryReader.open(writer);
  writer.close();

  IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
  searcher.setSimilarity(similarity);
  return searcher;
}
 
Example #8
Source File: RangeFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Build the example index. */
public void index() throws IOException {
  IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
      new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));

  // Add documents with a fake timestamp, 1000 sec before
  // "now", 2000 sec before "now", ...:
  for(int i=0;i<100;i++) {
    Document doc = new Document();
    long then = nowSec - i * 1000;
    // Add as doc values field, so we can compute range facets:
    doc.add(new NumericDocValuesField("timestamp", then));
    // Add as numeric field so we can drill-down:
    doc.add(new LongPoint("timestamp", then));
    indexWriter.addDocument(doc);
  }

  // Open near-real-time searcher
  searcher = new IndexSearcher(DirectoryReader.open(indexWriter));
  indexWriter.close();
}
 
Example #9
Source File: CodePatternSearcher.java    From SnowGraph with Apache License 2.0 6 votes vote down vote up
private static List<String> search(List<String> contents, String query, int n) throws IOException, ParseException {
    List<String> r=new ArrayList<>();
    Directory dir=new RAMDirectory();
    IndexWriter indexWriter=new IndexWriter(dir, new IndexWriterConfig(new EnglishAnalyzer()));
    for (String method:contents){
        Document document=new Document();
        document.add(new TextField("content",method, Field.Store.YES));
        indexWriter.addDocument(document);
    }
    indexWriter.close();
    QueryParser qp = new QueryParser("content", new EnglishAnalyzer());
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(dir));
    TopDocs topDocs = indexSearcher.search(qp.parse(query), n);
    for (ScoreDoc scoreDoc : topDocs.scoreDocs) {
        r.add(indexSearcher.doc(scoreDoc.doc).get("content"));
    }
    return r;
}
 
Example #10
Source File: BaseLockFactoryTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testStressLocks() throws Exception {
  Path tempPath = createTempDir();
  assumeFalse("cannot handle buggy Files.delete", TestUtil.hasWindowsFS(tempPath));

  Directory dir = getDirectory(tempPath);

  // First create a 1 doc index:
  IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.CREATE));
  addDoc(w);
  w.close();
  
  int numIterations = atLeast(20);
  WriterThread writer = new WriterThread(numIterations, dir);
  SearcherThread searcher = new SearcherThread(numIterations, dir);
  writer.start();
  searcher.start();

  writer.join();
  searcher.join();

  assertTrue("IndexWriter hit unexpected exceptions", !writer.hitException);
  assertTrue("IndexSearcher hit unexpected exceptions", !searcher.hitException);
  
  dir.close();
}
 
Example #11
Source File: QueryAutoStopWordAnalyzerTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = new ByteBuffersDirectory();
  appAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(appAnalyzer));
  int numDocs = 200;
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    String variedFieldValue = variedFieldValues[i % variedFieldValues.length];
    String repetitiveFieldValue = repetitiveFieldValues[i % repetitiveFieldValues.length];
    doc.add(new TextField("variedField", variedFieldValue, Field.Store.YES));
    doc.add(new TextField("repetitiveField", repetitiveFieldValue, Field.Store.YES));
    writer.addDocument(doc);
  }
  writer.close();
  reader = DirectoryReader.open(dir);
}
 
Example #12
Source File: LuceneIndexer.java    From MtgDesktopCompanion with GNU General Public License v3.0 6 votes vote down vote up
public void initIndex() throws IOException {
	
	if(dir==null)
		open();
	
	IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
	  				   iwc.setOpenMode(OpenMode.CREATE);
    IndexWriter indexWriter = new IndexWriter(dir, iwc);
    
	for(MagicCard mc : MTGControler.getInstance().getEnabled(MTGCardsProvider.class).listAllCards())
	{
		try {
			indexWriter.addDocument(toDocuments(mc));
		} catch (IllegalArgumentException e) {
			logger.error("Error indexing " + mc + " " + mc.getCurrentSet(),e);
		}
	}
	
	indexWriter.commit();
	indexWriter.close();
}
 
Example #13
Source File: Index.java    From dacapobench with Apache License 2.0 6 votes vote down vote up
/**
 * Index all text files under a directory.
 */
public void main(final File INDEX_DIR, final String[] args) throws IOException {
  IndexWriterConfig IWConfig = new IndexWriterConfig();
  IWConfig.setOpenMode (IndexWriterConfig.OpenMode.CREATE);
  IWConfig.setMergePolicy (new LogByteSizeMergePolicy());
  IndexWriter writer = new IndexWriter(FSDirectory.open(Paths.get(INDEX_DIR.getCanonicalPath())), IWConfig);
  for (int arg = 0; arg < args.length; arg++) {
    final File docDir = new File(args[arg]);
    if (!docDir.exists() || !docDir.canRead()) {
      System.out.println("Document directory '" + docDir.getAbsolutePath() + "' does not exist or is not readable, please check the path");
      throw new IOException("Cannot read from document directory");
    }

    indexDocs(writer, docDir);
    System.out.println("Optimizing...");
    writer.forceMerge(1);
  }
  writer.close();
}
 
Example #14
Source File: GroupByOptimizedIteratorTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testHighCardinalityRatioReturnsTrueForLowCardinality() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    String columnName = "x";
    for (int i = 0; i < 10; i++) {
        Document doc = new Document();
        BytesRef value = new BytesRef("1");
        doc.add(new Field(columnName, value, KeywordFieldMapper.Defaults.FIELD_TYPE.clone()));
        iw.addDocument(doc);
    }
    iw.commit();

    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    assertThat(
        GroupByOptimizedIterator.hasHighCardinalityRatio(() -> new Engine.Searcher("dummy", indexSearcher, () -> {}), "x"),
        is(false)
    );
}
 
Example #15
Source File: SpatialHelperTest.java    From geode-examples with Apache License 2.0 6 votes vote down vote up
@Test
public void queryFindsADocumentThatWasAdded() throws IOException {

  // Create an in memory lucene index to add a document to
  RAMDirectory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig());

  // Add a document to the lucene index
  Document document = new Document();
  document.add(new TextField("name", "name", Field.Store.YES));
  Field[] fields = SpatialHelper.getIndexableFields(-122.8515139, 45.5099231);
  for (Field field : fields) {
    document.add(field);
  }
  writer.addDocument(document);
  writer.commit();


  // Make sure a findWithin query locates the document
  Query query = SpatialHelper.findWithin(-122.8515239, 45.5099331, 1);
  SearcherManager searcherManager = new SearcherManager(writer, null);
  IndexSearcher searcher = searcherManager.acquire();
  TopDocs results = searcher.search(query, 100);
  assertEquals(1, results.totalHits);
}
 
Example #16
Source File: FastHdfsKeyValueDirectoryTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void testMultipleWritersOpenOnSameDirectory() throws IOException {
  IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer());
  FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path,
      "test_multiple"));
  IndexWriter writer1 = new IndexWriter(directory, config.clone());
  addDoc(writer1, getDoc(1));
  IndexWriter writer2 = new IndexWriter(directory, config.clone());
  addDoc(writer2, getDoc(2));
  writer1.close();
  writer2.close();

  DirectoryReader reader = DirectoryReader.open(directory);
  int maxDoc = reader.maxDoc();
  assertEquals(1, maxDoc);
  Document document = reader.document(0);
  assertEquals("2", document.get("id"));
  reader.close();
}
 
Example #17
Source File: TestLucene.java    From RedisDirectory with Apache License 2.0 6 votes vote down vote up
public void testRamDirectory() throws IOException {
    long start = System.currentTimeMillis();
    IndexWriterConfig indexWriterConfig = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(IndexWriterConfig
            .OpenMode.CREATE);
    RAMDirectory ramDirectory = new RAMDirectory();
    IndexWriter indexWriter = new IndexWriter(ramDirectory, indexWriterConfig);
    for (int i = 0; i < 10000000; i++) {
        indexWriter.addDocument(addDocument(i));
    }
    indexWriter.commit();
    indexWriter.close();
    long end = System.currentTimeMillis();
    log.error("RamDirectory consumes {}s!", (end - start) / 1000);
    start = System.currentTimeMillis();
    IndexSearcher indexSearcher = new IndexSearcher(DirectoryReader.open(ramDirectory));
    int total = 0;
    for (int i = 0; i < 10000000; i++) {
        TermQuery key1 = new TermQuery(new Term("key1", "key" + i));
        TopDocs search = indexSearcher.search(key1, 10);
        total += search.totalHits;
    }
    System.out.println(total);
    end = System.currentTimeMillis();
    log.error("RamDirectory search consumes {}ms!", (end - start));
}
 
Example #18
Source File: TestPerFieldPostingsFormat2.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testMergeUnusedPerFieldCodec() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig iwconf = newIndexWriterConfig(new MockAnalyzer(random()))
                               .setOpenMode(OpenMode.CREATE).setCodec(new MockCodec());
  IndexWriter writer = newWriter(dir, iwconf);
  addDocs(writer, 10);
  writer.commit();
  addDocs3(writer, 10);
  writer.commit();
  addDocs2(writer, 10);
  writer.commit();
  assertEquals(30, writer.getDocStats().maxDoc);
  TestUtil.checkIndex(dir);
  writer.forceMerge(1);
  assertEquals(30, writer.getDocStats().maxDoc);
  writer.close();
  dir.close();
}
 
Example #19
Source File: TestSearcherManager.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testEnsureOpen() throws Exception {
  Directory dir = newDirectory();
  new IndexWriter(dir, new IndexWriterConfig(null)).close();
  SearcherManager sm = new SearcherManager(dir, null);
  IndexSearcher s = sm.acquire();
  sm.close();
  
  // this should succeed;
  sm.release(s);
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.acquire();
  });
  
  // this should fail
  expectThrows(AlreadyClosedException.class, () -> {
    sm.maybeRefresh();
  });

  dir.close();
}
 
Example #20
Source File: TestPerfTasksLogic.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Test index creation logic
 */
public void testIndexAndSearchTasks() throws Exception {
  // 1. alg definition (required in every "logic" test)
  String algLines[] = {
      "ResetSystemErase",
      "CreateIndex",
      "{ AddDoc } : 1000",
      "ForceMerge(1)",
      "CloseIndex",
      "OpenReader",
      "{ CountingSearchTest } : 200",
      "CloseReader",
      "[ CountingSearchTest > : 70",
      "[ CountingSearchTest > : 9",
  };
  
  // 2. we test this value later
  CountingSearchTestTask.numSearches = 0;
  
  // 3. execute the algorithm  (required in every "logic" test)
  Benchmark benchmark = execBenchmark(algLines);

  // 4. test specific checks after the benchmark run completed.
  assertEquals("TestSearchTask was supposed to be called!",279,CountingSearchTestTask.numSearches);
  assertTrue("Index does not exist?...!", DirectoryReader.indexExists(benchmark.getRunData().getDirectory()));
  // now we should be able to open the index for write. 
  IndexWriter iw = new IndexWriter(benchmark.getRunData().getDirectory(),
      new IndexWriterConfig(new MockAnalyzer(random()))
          .setOpenMode(OpenMode.APPEND));
  iw.close();
  IndexReader ir = DirectoryReader.open(benchmark.getRunData().getDirectory());
  assertEquals("1000 docs were added to the index, this is what we expect to find!",1000,ir.numDocs());
  ir.close();
}
 
Example #21
Source File: IndexBilingualFiles.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
private void runIndexer() {
	if (Files.exists(INDEX_DIR)) {
     throw new IllegalArgumentException(
         "Cannot save index to '" + INDEX_DIR + "' directory, please delete it first");
	}

	Date start = new Date();
	try {
		final File docDir1 = new File(LANGUAGE1);
		final File docDir2 = new File(LANGUAGE2);

		IndexWriterConfig writerConfig = new IndexWriterConfig(new StandardAnalyzer());
		IndexWriter writer = new IndexWriter(FSDirectory.open(INDEX_DIR), writerConfig);
		System.out.println("Indexing to directory '" + INDEX_DIR + "'...");
		runDeepIndexer(docDir1, docDir2, writer);

		System.out.println("Optimizing...");
		writer.close();

		Date end = new Date();
		System.out.println(end.getTime() - start.getTime() + " total milliseconds");

	} catch (IOException e) {
		System.out.println(" caught a " + e.getClass() +
				"\n with message: " + e.getMessage());
	}
}
 
Example #22
Source File: LuceneIndexSettings.java    From sdudoc with MIT License 5 votes vote down vote up
/**
 * 创建磁盘目录
 */
public void createFSDirectory(String path) throws Exception {
	this.directory =  FSDirectory.open(new File(path));
	IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_44, this.analyzer);
	IndexWriter indexWriter = new IndexWriter(this.directory, indexWriterConfig);
	indexWriter.close();
}
 
Example #23
Source File: FunctionTestSetup.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected static void createIndex(boolean doMultiSegment) throws Exception {
  if (VERBOSE) {
    System.out.println("TEST: setUp");
  }
  // prepare a small index with just a few documents.
  dir = newDirectory();
  anlzr = new MockAnalyzer(random());
  IndexWriterConfig iwc = newIndexWriterConfig(anlzr).setMergePolicy(newLogMergePolicy());
  if (doMultiSegment) {
    iwc.setMaxBufferedDocs(TestUtil.nextInt(random(), 2, 7));
  }
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  // add docs not exactly in natural ID order, to verify we do check the order of docs by scores
  int remaining = N_DOCS;
  boolean done[] = new boolean[N_DOCS];
  int i = 0;
  while (remaining > 0) {
    if (done[i]) {
      throw new Exception("to set this test correctly N_DOCS=" + N_DOCS + " must be primary and greater than 2!");
    }
    addDoc(iw, i);
    done[i] = true;
    i = (i + 4) % N_DOCS;
    remaining --;
  }
  if (!doMultiSegment) {
    if (VERBOSE) {
      System.out.println("TEST: setUp full merge");
    }
    iw.forceMerge(1);
  }
  iw.close();
  if (VERBOSE) {
    System.out.println("TEST: setUp done close");
  }
}
 
Example #24
Source File: TestNRTCachingDirectory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void verifyCompiles() throws Exception {
  Analyzer analyzer = null;

  Directory fsDir = FSDirectory.open(createTempDir("verify"));
  NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 2.0, 25.0);
  IndexWriterConfig conf = new IndexWriterConfig(analyzer);
  IndexWriter writer = new IndexWriter(cachedFSDir, conf);
  writer.close();
  cachedFSDir.close();
}
 
Example #25
Source File: TestBlockPostingsFormat2.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newFSDirectory(createTempDir("testDFBlockSize"));
  IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
  iwc.setCodec(TestUtil.alwaysPostingsFormat(new Lucene50RWPostingsFormat()));
  iw = new RandomIndexWriter(random(), dir, iwc);
  iw.setDoRandomForceMerge(false); // we will ourselves
}
 
Example #26
Source File: IndexBuilder.java    From exhibitor with Apache License 2.0 5 votes vote down vote up
public void open() throws Exception
{
    if ( !directory.exists() && !directory.mkdirs() )
    {
        throw new IOException("Could not make: " + directory);
    }

    IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_35, new KeywordAnalyzer()).setOpenMode(IndexWriterConfig.OpenMode.CREATE);

    niofsDirectory = new NIOFSDirectory(directory, new SingleInstanceLockFactory());
    writer = new IndexWriter(niofsDirectory, conf);
}
 
Example #27
Source File: TestFieldSortOptimizationSkipping.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * test that even if a field is not indexed with points, optimized sort still works as expected,
 * although no optimization will be run
 */
public void testLongSortOptimizationOnFieldNotIndexedWithPoints() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
  final int numDocs = atLeast(100);
  // my_field is not indexed with points
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = new Document();
    doc.add(new NumericDocValuesField("my_field", i));
    writer.addDocument(doc);
  }
  final IndexReader reader = DirectoryReader.open(writer);
  IndexSearcher searcher = new IndexSearcher(reader);
  final SortField sortField = new SortField("my_field", SortField.Type.LONG);
  final Sort sort = new Sort(sortField);
  final int numHits = 3;
  final int totalHitsThreshold = 3;

  final TopFieldCollector collector = TopFieldCollector.create(sort, numHits, null, totalHitsThreshold);
  searcher.search(new MatchAllDocsQuery(), collector);
  TopDocs topDocs = collector.topDocs();
  assertEquals(topDocs.scoreDocs.length, numHits);  // sort still works and returns expected number of docs
  for (int i = 0; i < numHits; i++) {
    FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i];
    assertEquals(i, ((Long) fieldDoc.fields[0]).intValue()); // returns expected values
  }
  assertEquals(topDocs.totalHits.value, numDocs); // assert that all documents were collected => optimization was not run

  writer.close();
  reader.close();
  dir.close();
}
 
Example #28
Source File: SharedMergeSchedulerThroughputTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
public void test() throws IOException {
  MiniCluster miniCluster = new MiniCluster();
  miniCluster.startDfs("./tmp/hdfs");
  IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43));
  SharedMergeScheduler sharedMergeScheduler = new SharedMergeScheduler(10);
  conf.setMergeScheduler(sharedMergeScheduler.getMergeScheduler());
  Configuration configuration = new Configuration();
  URI fileSystemUri = miniCluster.getFileSystemUri();
  Path path = new Path(fileSystemUri.toString() + "/merge-test");
  FileSystem fileSystem = path.getFileSystem(configuration);
  fileSystem.delete(path, true);
  HdfsDirectory directory = new HdfsDirectory(configuration, path);
  BlurConfiguration blurConfiguration = new BlurConfiguration();
  BlockCacheDirectoryFactoryV2 factory = new BlockCacheDirectoryFactoryV2(blurConfiguration, 1000000l);

  Directory cdir = factory.newDirectory("t", "s", directory, null);
  IndexWriter writer = new IndexWriter(cdir, conf);
  Random random = new Random(1);
  StringBuilder stringBuilder = new StringBuilder();
  long s = System.nanoTime();
  for (int i = 0; i < 250000; i++) {
    if (i % 5000 == 0) {
      System.out.println(i);
    }
    stringBuilder.setLength(0);
    for (int w = 0; w < 2000; w++) {
      stringBuilder.append(Integer.toString(random.nextInt(100000))).append(' ');
    }
    Document document = new Document();
    document.add(new TextField("body", stringBuilder.toString(), Store.YES));
    writer.addDocument(document);
  }
  writer.close(true);
  sharedMergeScheduler.close();
  factory.close();
  long e = System.nanoTime();
  System.out.println("Total Time [" + (e - s) / 1000000.0 + " ms]");
  miniCluster.shutdownDfs();
}
 
Example #29
Source File: TextSearchQueriesTest.java    From incubator-pinot with Apache License 2.0 5 votes vote down vote up
@Test
public void testMultiThreadedLuceneRealtime() throws Exception {
  File indexFile = new File(INDEX_DIR.getPath() + "/realtime-test3.index");
  Directory indexDirectory = FSDirectory.open(indexFile.toPath());
  StandardAnalyzer standardAnalyzer = new StandardAnalyzer();
  // create and open a writer
  IndexWriterConfig indexWriterConfig = new IndexWriterConfig(standardAnalyzer);
  indexWriterConfig.setRAMBufferSizeMB(500);
  IndexWriter indexWriter = new IndexWriter(indexDirectory, indexWriterConfig);

  // create an NRT index reader
  SearcherManager searcherManager = new SearcherManager(indexWriter, false, false, null);

  // background thread to refresh NRT reader
  ControlledRealTimeReopenThread controlledRealTimeReopenThread =
      new ControlledRealTimeReopenThread(indexWriter, searcherManager, 0.01, 0.01);
  controlledRealTimeReopenThread.start();

  // start writer and reader
  Thread writer = new Thread(new RealtimeWriter(indexWriter));
  Thread realtimeReader = new Thread(new RealtimeReader(searcherManager, standardAnalyzer));

  writer.start();
  realtimeReader.start();

  writer.join();
  realtimeReader.join();
  controlledRealTimeReopenThread.join();
}
 
Example #30
Source File: TestPerFieldPostingsFormat2.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private IndexWriter newWriter(Directory dir, IndexWriterConfig conf)
    throws IOException {
  LogDocMergePolicy logByteSizeMergePolicy = new LogDocMergePolicy();
  logByteSizeMergePolicy.setNoCFSRatio(0.0); // make sure we use plain
  // files
  conf.setMergePolicy(logByteSizeMergePolicy);

  final IndexWriter writer = new IndexWriter(dir, conf);
  return writer;
}