Java Code Examples for org.apache.lucene.index.IndexWriterConfig#setMergeScheduler()
The following examples show how to use
org.apache.lucene.index.IndexWriterConfig#setMergeScheduler() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestMergeSchedulerExternal.java From lucene-solr with Apache License 2.0 | 6 votes |
public void testCustomMergeScheduler() throws Exception { // we don't really need to execute anything, just to make sure the custom MS // compiles. But ensure that it can be used as well, e.g., no other hidden // dependencies or something. Therefore, don't use any random API ! Directory dir = new ByteBuffersDirectory(); IndexWriterConfig conf = new IndexWriterConfig(null); conf.setMergeScheduler(new ReportingMergeScheduler()); IndexWriter writer = new IndexWriter(dir, conf); writer.addDocument(new Document()); writer.commit(); // trigger flush writer.addDocument(new Document()); writer.commit(); // trigger flush writer.forceMerge(1); writer.close(); dir.close(); }
Example 2
Source File: BaseShapeTestCase.java From lucene-solr with Apache License 2.0 | 6 votes |
private void verify(Object... shapes) throws Exception { IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setMergeScheduler(new SerialMergeScheduler()); int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < shapes.length / 100) { iwc.setMaxBufferedDocs(shapes.length / 100); } Directory dir; if (shapes.length > 1000) { dir = newFSDirectory(createTempDir(getClass().getSimpleName())); } else { dir = newDirectory(); } IndexWriter w = new IndexWriter(dir, iwc); // index random polygons indexRandomShapes(w, shapes); // query testing final IndexReader reader = DirectoryReader.open(w); // test random bbox queries verifyRandomQueries(reader, shapes); IOUtils.close(w, reader, dir); }
Example 3
Source File: test.java From vscode-extension with MIT License | 5 votes |
private IndexWriterConfig getIndexWriterConfig() { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); iwc.setIndexDeletionPolicy(combinedDeletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); // Give us the opportunity to upgrade old segments while performing // background merges MergePolicy mergePolicy = config().getMergePolicy(); // always configure soft-deletes field so an engine with soft-deletes disabled can open a Lucene index with soft-deletes. iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD); if (softDeleteEnabled) { mergePolicy = new RecoverySourcePruneMergePolicy(SourceFieldMapper.RECOVERY_SOURCE_NAME, softDeletesPolicy::getRetentionQuery, new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery, mergePolicy)); } iwc.setMergePolicy(new ElasticsearchMergePolicy(mergePolicy)); iwc.setSimilarity(engineConfig.getSimilarity()); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh if (config().getIndexSort() != null) { iwc.setIndexSort(config().getIndexSort()); } return iwc; }
Example 4
Source File: SimpleIndexManager.java From localization_nifi with Apache License 2.0 | 5 votes |
private IndexWriterCount createWriter(final File indexDirectory) throws IOException { final List<Closeable> closeables = new ArrayList<>(); final Directory directory = FSDirectory.open(indexDirectory); closeables.add(directory); try { final Analyzer analyzer = new StandardAnalyzer(); closeables.add(analyzer); final IndexWriterConfig config = new IndexWriterConfig(LuceneUtil.LUCENE_VERSION, analyzer); final ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler(); final int mergeThreads = repoConfig.getConcurrentMergeThreads(); mergeScheduler.setMaxMergesAndThreads(mergeThreads, mergeThreads); config.setMergeScheduler(mergeScheduler); final IndexWriter indexWriter = new IndexWriter(directory, config); final EventIndexWriter eventIndexWriter = new LuceneEventIndexWriter(indexWriter, indexDirectory); final IndexWriterCount writerCount = new IndexWriterCount(eventIndexWriter, analyzer, directory, 1, false); logger.debug("Providing new index writer for {}", indexDirectory); return writerCount; } catch (final IOException ioe) { for (final Closeable closeable : closeables) { try { closeable.close(); } catch (final IOException ioe2) { ioe.addSuppressed(ioe2); } } throw ioe; } }
Example 5
Source File: LindenConfig.java From linden with Apache License 2.0 | 5 votes |
public IndexWriterConfig createIndexWriterConfig() throws IOException { IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LATEST, getIndexAnalyzerInstance()); indexWriterConfig.setRAMBufferSizeMB(48); MergePolicy mergePolicy = getPluginManager().getInstance(LindenConfigBuilder.MERGE_POLICY, MergePolicy.class); if (mergePolicy != null) { indexWriterConfig.setMergePolicy(mergePolicy); } LOGGER.info("Merge policy : {}", mergePolicy == null ? "Default" : mergePolicy); ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler(); cms.setMaxMergesAndThreads(8, 1); indexWriterConfig.setMergeScheduler(cms); return indexWriterConfig; }
Example 6
Source File: SolrIndexConfig.java From lucene-solr with Apache License 2.0 | 5 votes |
public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException { IndexSchema schema = core.getLatestSchema(); IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core)); if (maxBufferedDocs != -1) iwc.setMaxBufferedDocs(maxBufferedDocs); if (ramBufferSizeMB != -1) iwc.setRAMBufferSizeMB(ramBufferSizeMB); if (ramPerThreadHardLimitMB != -1) { iwc.setRAMPerThreadHardLimitMB(ramPerThreadHardLimitMB); } iwc.setSimilarity(schema.getSimilarity()); MergePolicy mergePolicy = buildMergePolicy(core.getResourceLoader(), schema); iwc.setMergePolicy(mergePolicy); MergeScheduler mergeScheduler = buildMergeScheduler(core.getResourceLoader()); iwc.setMergeScheduler(mergeScheduler); iwc.setInfoStream(infoStream); if (mergePolicy instanceof SortingMergePolicy) { Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort(); iwc.setIndexSort(indexSort); } iwc.setUseCompoundFile(useCompoundFile); if (mergedSegmentWarmerInfo != null) { // TODO: add infostream -> normal logging system (there is an issue somewhere) @SuppressWarnings({"rawtypes"}) IndexReaderWarmer warmer = core.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className, IndexReaderWarmer.class, null, new Class[] { InfoStream.class }, new Object[] { iwc.getInfoStream() }); iwc.setMergedSegmentWarmer(warmer); } return iwc; }
Example 7
Source File: SharedMergeSchedulerThroughputTest.java From incubator-retired-blur with Apache License 2.0 | 5 votes |
public void test() throws IOException { MiniCluster miniCluster = new MiniCluster(); miniCluster.startDfs("./tmp/hdfs"); IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new StandardAnalyzer(Version.LUCENE_43)); SharedMergeScheduler sharedMergeScheduler = new SharedMergeScheduler(10); conf.setMergeScheduler(sharedMergeScheduler.getMergeScheduler()); Configuration configuration = new Configuration(); URI fileSystemUri = miniCluster.getFileSystemUri(); Path path = new Path(fileSystemUri.toString() + "/merge-test"); FileSystem fileSystem = path.getFileSystem(configuration); fileSystem.delete(path, true); HdfsDirectory directory = new HdfsDirectory(configuration, path); BlurConfiguration blurConfiguration = new BlurConfiguration(); BlockCacheDirectoryFactoryV2 factory = new BlockCacheDirectoryFactoryV2(blurConfiguration, 1000000l); Directory cdir = factory.newDirectory("t", "s", directory, null); IndexWriter writer = new IndexWriter(cdir, conf); Random random = new Random(1); StringBuilder stringBuilder = new StringBuilder(); long s = System.nanoTime(); for (int i = 0; i < 250000; i++) { if (i % 5000 == 0) { System.out.println(i); } stringBuilder.setLength(0); for (int w = 0; w < 2000; w++) { stringBuilder.append(Integer.toString(random.nextInt(100000))).append(' '); } Document document = new Document(); document.add(new TextField("body", stringBuilder.toString(), Store.YES)); writer.addDocument(document); } writer.close(true); sharedMergeScheduler.close(); factory.close(); long e = System.nanoTime(); System.out.println("Total Time [" + (e - s) / 1000000.0 + " ms]"); miniCluster.shutdownDfs(); }
Example 8
Source File: StandardIndexManager.java From nifi with Apache License 2.0 | 5 votes |
private IndexWriterCount createWriter(final File indexDirectory) throws IOException { final List<Closeable> closeables = new ArrayList<>(); final Directory directory = FSDirectory.open(indexDirectory.toPath()); closeables.add(directory); try { final Analyzer analyzer = new StandardAnalyzer(); closeables.add(analyzer); final IndexWriterConfig config = new IndexWriterConfig(analyzer); final ConcurrentMergeScheduler mergeScheduler = new ConcurrentMergeScheduler(); final int mergeThreads = repoConfig.getConcurrentMergeThreads(); mergeScheduler.setMaxMergesAndThreads(mergeThreads, mergeThreads); config.setMergeScheduler(mergeScheduler); final IndexWriter indexWriter = new IndexWriter(directory, config); final EventIndexWriter eventIndexWriter = new LuceneEventIndexWriter(indexWriter, indexDirectory); final IndexWriterCount writerCount = new IndexWriterCount(eventIndexWriter, analyzer, directory, 1, false); logger.debug("Providing new index writer for {}", indexDirectory); return writerCount; } catch (final IOException ioe) { for (final Closeable closeable : closeables) { try { closeable.close(); } catch (final IOException ioe2) { ioe.addSuppressed(ioe2); } } throw ioe; } }
Example 9
Source File: InternalEngine.java From crate with Apache License 2.0 | 5 votes |
private IndexWriterConfig getIndexWriterConfig() { final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer()); iwc.setCommitOnClose(false); // we by default don't commit on close iwc.setOpenMode(IndexWriterConfig.OpenMode.APPEND); iwc.setReaderAttributes(getReaderAttributes(store.directory())); iwc.setIndexDeletionPolicy(combinedDeletionPolicy); // with tests.verbose, lucene sets this up: plumb to align with filesystem stream boolean verbose = false; try { verbose = Boolean.parseBoolean(System.getProperty("tests.verbose")); } catch (Exception ignore) { // ignored } iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger)); iwc.setMergeScheduler(mergeScheduler); // Give us the opportunity to upgrade old segments while performing // background merges MergePolicy mergePolicy = config().getMergePolicy(); // always configure soft-deletes field so an engine with soft-deletes disabled can open a Lucene index with soft-deletes. iwc.setSoftDeletesField(Lucene.SOFT_DELETES_FIELD); if (softDeleteEnabled) { mergePolicy = new RecoverySourcePruneMergePolicy(SourceFieldMapper.RECOVERY_SOURCE_NAME, softDeletesPolicy::getRetentionQuery, new SoftDeletesRetentionMergePolicy(Lucene.SOFT_DELETES_FIELD, softDeletesPolicy::getRetentionQuery, mergePolicy)); } iwc.setMergePolicy(new ElasticsearchMergePolicy(mergePolicy)); iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac()); iwc.setCodec(engineConfig.getCodec()); iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh return iwc; }
Example 10
Source File: DocumentSearcher.java From meghanada-server with GNU General Public License v3.0 | 4 votes |
private static IndexWriterConfig createIndexConfig(Analyzer analyzer) { IndexWriterConfig config = new IndexWriterConfig(analyzer); config.setMergeScheduler(new ConcurrentMergeScheduler()); config.setCodec(new Lucene70CodecWithNoFieldCompression()); return config; }
Example 11
Source File: TestNearest.java From lucene-solr with Apache License 2.0 | 4 votes |
public void testNearestNeighborRandom() throws Exception { int numPoints = atLeast(1000); Directory dir; if (numPoints > 100000) { dir = newFSDirectory(createTempDir(getClass().getSimpleName())); } else { dir = newDirectory(); } double[] lats = new double[numPoints]; double[] lons = new double[numPoints]; IndexWriterConfig iwc = getIndexWriterConfig(); iwc.setMergePolicy(newLogMergePolicy()); iwc.setMergeScheduler(new SerialMergeScheduler()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); for(int id=0;id<numPoints;id++) { lats[id] = quantizeLat(GeoTestUtil.nextLatitude()); lons[id] = quantizeLon(GeoTestUtil.nextLongitude()); Document doc = new Document(); doc.add(new LatLonPoint("point", lats[id], lons[id])); doc.add(new LatLonDocValuesField("point", lats[id], lons[id])); doc.add(new StoredField("id", id)); w.addDocument(doc); } if (random().nextBoolean()) { w.forceMerge(1); } DirectoryReader r = w.getReader(); if (VERBOSE) { System.out.println("TEST: reader=" + r); } // can't wrap because we require Lucene60PointsFormat directly but e.g. ParallelReader wraps with its own points impl: IndexSearcher s = newSearcher(r, false); int iters = atLeast(100); for(int iter=0;iter<iters;iter++) { if (VERBOSE) { System.out.println("\nTEST: iter=" + iter); } double pointLat = GeoTestUtil.nextLatitude(); double pointLon = GeoTestUtil.nextLongitude(); // dumb brute force search to get the expected result: FieldDoc[] expectedHits = new FieldDoc[lats.length]; for(int id=0;id<lats.length;id++) { double distance = SloppyMath.haversinMeters(pointLat, pointLon, lats[id], lons[id]); FieldDoc hit = new FieldDoc(id, 0.0f, new Object[] {Double.valueOf(distance)}); expectedHits[id] = hit; } Arrays.sort(expectedHits, new Comparator<FieldDoc>() { @Override public int compare(FieldDoc a, FieldDoc b) { int cmp = Double.compare(((Double) a.fields[0]).doubleValue(), ((Double) b.fields[0]).doubleValue()); if (cmp != 0) { return cmp; } // tie break by smaller docID: return a.doc - b.doc; } }); int topN = TestUtil.nextInt(random(), 1, lats.length); if (VERBOSE) { System.out.println("\nhits for pointLat=" + pointLat + " pointLon=" + pointLon); } // Also test with MatchAllDocsQuery, sorting by distance: TopFieldDocs fieldDocs = s.search(new MatchAllDocsQuery(), topN, new Sort(LatLonDocValuesField.newDistanceSort("point", pointLat, pointLon))); ScoreDoc[] hits = LatLonPointPrototypeQueries.nearest(s, "point", pointLat, pointLon, topN).scoreDocs; for(int i=0;i<topN;i++) { FieldDoc expected = expectedHits[i]; FieldDoc expected2 = (FieldDoc) fieldDocs.scoreDocs[i]; FieldDoc actual = (FieldDoc) hits[i]; Document actualDoc = r.document(actual.doc); if (VERBOSE) { System.out.println("hit " + i); System.out.println(" expected id=" + expected.doc+ " lat=" + lats[expected.doc] + " lon=" + lons[expected.doc] + " distance=" + ((Double) expected.fields[0]).doubleValue() + " meters"); System.out.println(" actual id=" + actualDoc.getField("id") + " distance=" + actual.fields[0] + " meters"); } assertEquals(expected.doc, actual.doc); assertEquals(((Double) expected.fields[0]).doubleValue(), ((Double) actual.fields[0]).doubleValue(), 0.0); assertEquals(expected2.doc, actual.doc); assertEquals(((Double) expected2.fields[0]).doubleValue(), ((Double) actual.fields[0]).doubleValue(), 0.0); } } r.close(); w.close(); dir.close(); }
Example 12
Source File: BaseGeoPointTestCase.java From lucene-solr with Apache License 2.0 | 4 votes |
public void testRectBoundariesAreInclusive() throws Exception { Rectangle rect; // TODO: why this dateline leniency??? while (true) { rect = nextBox(); if (rect.crossesDateline() == false) { break; } } // this test works in quantized space: for testing inclusiveness of exact edges it must be aware of index-time quantization! rect = new Rectangle(quantizeLat(rect.minLat), quantizeLat(rect.maxLat), quantizeLon(rect.minLon), quantizeLon(rect.maxLon)); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); // Else seeds may not reproduce: iwc.setMergeScheduler(new SerialMergeScheduler()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); for(int x=0;x<3;x++) { double lat; if (x == 0) { lat = rect.minLat; } else if (x == 1) { lat = quantizeLat((rect.minLat+rect.maxLat)/2.0); } else { lat = rect.maxLat; } for(int y=0;y<3;y++) { double lon; if (y == 0) { lon = rect.minLon; } else if (y == 1) { if (x == 1) { continue; } lon = quantizeLon((rect.minLon+rect.maxLon)/2.0); } else { lon = rect.maxLon; } Document doc = new Document(); addPointToDoc(FIELD_NAME, doc, lat, lon); w.addDocument(doc); } } IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); // exact edge cases assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, rect.maxLon))); // expand 1 ulp in each direction if possible and test a slightly larger box! if (rect.minLat != -90) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, Math.nextDown(rect.minLat), rect.maxLat, rect.minLon, rect.maxLon))); } if (rect.maxLat != 90) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, Math.nextUp(rect.maxLat), rect.minLon, rect.maxLon))); } if (rect.minLon != -180) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, Math.nextDown(rect.minLon), rect.maxLon))); } if (rect.maxLon != 180) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minLat, rect.maxLat, rect.minLon, Math.nextUp(rect.maxLon)))); } // now shrink 1 ulp in each direction if possible: it should not include bogus stuff // we can't shrink if values are already at extremes, and // we can't do this if rectangle is actually a line or we will create a cross-dateline query if (rect.minLat != 90 && rect.maxLat != -90 && rect.minLon != 80 && rect.maxLon != -180 && rect.minLon != rect.maxLon) { // note we put points on "sides" not just "corners" so we just shrink all 4 at once for now: it should exclude all points! assertEquals(0, s.count(newRectQuery(FIELD_NAME, Math.nextUp(rect.minLat), Math.nextDown(rect.maxLat), Math.nextUp(rect.minLon), Math.nextDown(rect.maxLon)))); } r.close(); w.close(); dir.close(); }
Example 13
Source File: BaseXYPointTestCase.java From lucene-solr with Apache License 2.0 | 4 votes |
public void testRectBoundariesAreInclusive() throws Exception { XYRectangle rect = ShapeTestUtil.nextBox(random()); Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(); // Else seeds may not reproduce: iwc.setMergeScheduler(new SerialMergeScheduler()); RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc); for(int i = 0; i < 3; i++) { float y; if (i == 0) { y = rect.minY; } else if (i == 1) { y = (float) (((double) rect.minY + rect.maxY) / 2.0); } else { y = rect.maxY; } for(int j = 0; j < 3; j++) { float x; if (j == 0) { x = rect.minX; } else if (j == 1) { if (i == 1) { continue; } x = (float) (((double) rect.minX + rect.maxX) / 2.0); } else { x = rect.maxX; } Document doc = new Document(); addPointToDoc(FIELD_NAME, doc, x, y); w.addDocument(doc); } } IndexReader r = w.getReader(); IndexSearcher s = newSearcher(r, false); // exact edge cases assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minX, rect.maxX, rect.minY, rect.maxY))); // expand 1 ulp in each direction if possible and test a slightly larger box! if (rect.minX != -Float.MAX_VALUE) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, Math.nextDown(rect.minX), rect.maxX, rect.minY, rect.maxY))); } if (rect.maxX != Float.MAX_VALUE) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minX, Math.nextUp(rect.maxX), rect.minY, rect.maxY))); } if (rect.minY != -Float.MAX_VALUE) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minX, rect.maxX, Math.nextDown(rect.minY), rect.maxY))); } if (rect.maxY != Float.MAX_VALUE) { assertEquals(8, s.count(newRectQuery(FIELD_NAME, rect.minX, rect.maxX, rect.minY, Math.nextUp(rect.maxY)))); } r.close(); w.close(); dir.close(); }
Example 14
Source File: TestTopFieldCollectorEarlyTermination.java From lucene-solr with Apache License 2.0 | 4 votes |
private void createRandomIndex(boolean singleSortedSegment) throws IOException { dir = newDirectory(); numDocs = atLeast(150); final int numTerms = TestUtil.nextInt(random(), 1, numDocs / 5); Set<String> randomTerms = new HashSet<>(); while (randomTerms.size() < numTerms) { randomTerms.add(TestUtil.randomSimpleString(random())); } terms = new ArrayList<>(randomTerms); final long seed = random().nextLong(); final IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(new Random(seed))); if (iwc.getMergePolicy() instanceof MockRandomMergePolicy) { // MockRandomMP randomly wraps the leaf readers which makes merging angry iwc.setMergePolicy(newTieredMergePolicy()); } iwc.setMergeScheduler(new SerialMergeScheduler()); // for reproducible tests iwc.setIndexSort(sort); iw = new RandomIndexWriter(new Random(seed), dir, iwc); iw.setDoRandomForceMerge(false); // don't do this, it may happen anyway with MockRandomMP for (int i = 0; i < numDocs; ++i) { final Document doc = randomDocument(); iw.addDocument(doc); if (i == numDocs / 2 || (i != numDocs - 1 && random().nextInt(8) == 0)) { iw.commit(); } if (random().nextInt(15) == 0) { final String term = RandomPicks.randomFrom(random(), terms); iw.deleteDocuments(new Term("s", term)); } } if (singleSortedSegment) { iw.forceMerge(1); } else if (random().nextBoolean()) { iw.forceMerge(FORCE_MERGE_MAX_SEGMENT_COUNT); } reader = iw.getReader(); if (reader.numDocs() == 0) { iw.addDocument(new Document()); reader.close(); reader = iw.getReader(); } }
Example 15
Source File: TestLatLonShape.java From lucene-solr with Apache License 2.0 | 4 votes |
/** test we can search for a point with a large number of vertices*/ public void testLargeVertexPolygon() throws Exception { int numVertices = TEST_NIGHTLY ? TestUtil.nextInt(random(), 200000, 500000) : TestUtil.nextInt(random(), 20000, 50000); IndexWriterConfig iwc = newIndexWriterConfig(); iwc.setMergeScheduler(new SerialMergeScheduler()); int mbd = iwc.getMaxBufferedDocs(); if (mbd != -1 && mbd < numVertices/100) { iwc.setMaxBufferedDocs(numVertices/100); } Directory dir = newFSDirectory(createTempDir(getClass().getSimpleName())); IndexWriter writer = new IndexWriter(dir, iwc); // add a random polygon without a hole Polygon p = GeoTestUtil.createRegularPolygon(0, 90, atLeast(1000000), numVertices); Document document = new Document(); addPolygonsToDoc(FIELDNAME, document, p); writer.addDocument(document); // add a random polygon with a hole Polygon inner = new Polygon(new double[] {-1d, -1d, 1d, 1d, -1d}, new double[] {-91d, -89d, -89d, -91.0, -91.0}); Polygon outer = GeoTestUtil.createRegularPolygon(0, -90, atLeast(1000000), numVertices); document = new Document(); addPolygonsToDoc(FIELDNAME, document, new Polygon(outer.getPolyLats(), outer.getPolyLons(), inner)); writer.addDocument(document); ////// search ///// // search an intersecting bbox IndexReader reader = DirectoryReader.open(writer); writer.close(); IndexSearcher searcher = newSearcher(reader); Query q = newRectQuery(FIELDNAME, -1d, 1d, p.minLon, p.maxLon); assertEquals(1, searcher.count(q)); // search a disjoint bbox q = newRectQuery(FIELDNAME, p.minLat-1d, p.minLat+1, p.minLon-1d, p.minLon+1d); assertEquals(0, searcher.count(q)); // search a bbox in the hole q = newRectQuery(FIELDNAME, inner.minLat + 1e-6, inner.maxLat - 1e-6, inner.minLon + 1e-6, inner.maxLon - 1e-6); assertEquals(0, searcher.count(q)); IOUtils.close(reader, dir); }
Example 16
Source File: FastHdfsKeyValueDirectoryTest.java From incubator-retired-blur with Apache License 2.0 | 4 votes |
@Test public void testMulipleCommitsAndReopens() throws IOException { IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_43, new KeywordAnalyzer()); conf.setMergeScheduler(new SerialMergeScheduler()); TieredMergePolicy mergePolicy = (TieredMergePolicy) conf.getMergePolicy(); mergePolicy.setUseCompoundFile(false); Set<String> fileSet = new TreeSet<String>(); long seed = new Random().nextLong(); System.out.println("Seed:" + seed); Random random = new Random(seed); int docCount = 0; int passes = 10; byte[] segmentsGenContents = null; for (int run = 0; run < passes; run++) { final FastHdfsKeyValueDirectory directory = new FastHdfsKeyValueDirectory(false, _timer, _configuration, new Path(_path, "test_multiple_commits_reopens")); if (segmentsGenContents != null) { byte[] segmentsGenContentsCurrent = readSegmentsGen(directory); assertTrue(Arrays.equals(segmentsGenContents, segmentsGenContentsCurrent)); } assertFiles(fileSet, run, -1, directory); assertEquals(docCount, getDocumentCount(directory)); IndexWriter writer = new IndexWriter(directory, conf.clone()); int numberOfCommits = random.nextInt(100); for (int i = 0; i < numberOfCommits; i++) { assertFiles(fileSet, run, i, directory); addDocuments(writer, random.nextInt(100)); // Before Commit writer.commit(); // After Commit // Set files after commit { fileSet.clear(); List<IndexCommit> listCommits = DirectoryReader.listCommits(directory); assertEquals(1, listCommits.size()); IndexCommit indexCommit = listCommits.get(0); fileSet.addAll(indexCommit.getFileNames()); } segmentsGenContents = readSegmentsGen(directory); } docCount = getDocumentCount(directory); } }