org.apache.lucene.index.SegmentInfos Java Examples
The following examples show how to use
org.apache.lucene.index.SegmentInfos.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CommitsImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public Map<String, String> getSegmentDiagnostics(long commitGen, String name) throws LukeException { try { SegmentInfos infos = findSegmentInfos(commitGen); if (infos == null) { return Collections.emptyMap(); } return infos.asList().stream() .filter(seg -> seg.info.name.equals(name)) .findAny() .map(seg -> seg.info.getDiagnostics()) .orElse(Collections.emptyMap()); } catch (IOException e) { throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e); } }
Example #2
Source File: SegmentsInfoRequestHandlerTest.java From lucene-solr with Apache License 2.0 | 6 votes |
@Test public void testFieldInfo() throws Exception { String[] segmentNamePatterns = new String[NUM_SEGMENTS]; h.getCore().withSearcher((searcher) -> { int i = 0; for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) { assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS); segmentNamePatterns[i] = "boolean(//lst[@name='segments']/lst[@name='" + sInfo.info.name + "']/lst[@name='fields']/lst[@name='id']/str[@name='flags'])"; i++; } return null; }); assertQ("Unexpected field infos returned", req("qt","/admin/segments", "fieldInfo", "true"), segmentNamePatterns); }
Example #3
Source File: SegmentsInfoRequestHandlerTest.java From lucene-solr with Apache License 2.0 | 6 votes |
@Test public void testSegmentNames() throws IOException { String[] segmentNamePatterns = new String[NUM_SEGMENTS]; h.getCore().withSearcher((searcher) -> { int i = 0; for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) { assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS); segmentNamePatterns[i] = "//lst[@name='segments']/lst/str[@name='name'][.='" + sInfo.info.name + "']"; i++; } return null; }); assertQ("Unexpected segment names returned", req("qt","/admin/segments"), segmentNamePatterns); }
Example #4
Source File: SegmentsInfoRequestHandler.java From lucene-solr with Apache License 2.0 | 6 votes |
private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException { SimpleOrderedMap<Object> result = new SimpleOrderedMap<>(); RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore()); try { IndexWriter indexWriter = refCounted.get(); if (indexWriter instanceof SolrIndexWriter) { result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges()); } //get chosen merge policy MergePolicy mp = indexWriter.getConfig().getMergePolicy(); //Find merges MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter); if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) { for (OneMerge merge : findMerges.merges) { //TODO: add merge grouping for (SegmentCommitInfo mergeSegmentInfo : merge.segments) { mergeCandidates.add(mergeSegmentInfo.info.name); } } } return result; } finally { refCounted.decref(); } }
Example #5
Source File: TestSearcherTaxonomyManager.java From lucene-solr with Apache License 2.0 | 6 votes |
public void testExceptionDuringRefresh() throws Exception { Directory indexDir = newDirectory(); Directory taxoDir = newDirectory(); IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random()))); DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir); w.commit(); tw.commit(); SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null); tw.addCategory(new FacetLabel("a", "b")); w.addDocument(new Document()); tw.commit(); w.commit(); // intentionally corrupt the taxo index: SegmentInfos infos = SegmentInfos.readLatestCommit(taxoDir); taxoDir.deleteFile(infos.getSegmentsFileName()); expectThrows(IndexNotFoundException.class, mgr::maybeRefreshBlocking); IOUtils.close(w, tw, mgr, indexDir, taxoDir); }
Example #6
Source File: CommitsImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public List<Segment> getSegments(long commitGen) throws LukeException { try { SegmentInfos infos = findSegmentInfos(commitGen); if (infos == null) { return Collections.emptyList(); } return infos.asList().stream() .map(Segment::of) .sorted(Comparator.comparing(Segment::getName)) .collect(Collectors.toList()); } catch (IOException e) { throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e); } }
Example #7
Source File: CommitsImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public Map<String, String> getSegmentAttributes(long commitGen, String name) throws LukeException { try { SegmentInfos infos = findSegmentInfos(commitGen); if (infos == null) { return Collections.emptyMap(); } return infos.asList().stream() .filter(seg -> seg.info.name.equals(name)) .findAny() .map(seg -> seg.info.getAttributes()) .orElse(Collections.emptyMap()); } catch (IOException e) { throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e); } }
Example #8
Source File: CommitsImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public Optional<Codec> getSegmentCodec(long commitGen, String name) throws LukeException { try { SegmentInfos infos = findSegmentInfos(commitGen); if (infos == null) { return Optional.empty(); } return infos.asList().stream() .filter(seg -> seg.info.name.equals(name)) .findAny() .map(seg -> seg.info.getCodec()); } catch (IOException e) { throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e); } }
Example #9
Source File: TestDirectoryTaxonomyWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
@Test public void testPrepareCommitNoEmptyCommits() throws Exception { // LUCENE-4972: DTW used to create empty commits even if no changes were made Directory dir = newDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.addCategory(new FacetLabel("a")); taxoWriter.prepareCommit(); taxoWriter.commit(); long gen1 = SegmentInfos.getLastCommitGeneration(dir); taxoWriter.prepareCommit(); taxoWriter.commit(); long gen2 = SegmentInfos.getLastCommitGeneration(dir); assertEquals("empty commit should not have changed the index", gen1, gen2); taxoWriter.close(); dir.close(); }
Example #10
Source File: TestDirectoryTaxonomyWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
@Test public void testCloseNoEmptyCommits() throws Exception { // LUCENE-4972: DTW used to create empty commits even if no changes were made Directory dir = newDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.addCategory(new FacetLabel("a")); taxoWriter.commit(); long gen1 = SegmentInfos.getLastCommitGeneration(dir); taxoWriter.close(); long gen2 = SegmentInfos.getLastCommitGeneration(dir); assertEquals("empty commit should not have changed the index", gen1, gen2); taxoWriter.close(); dir.close(); }
Example #11
Source File: Engine.java From Elasticsearch with Apache License 2.0 | 6 votes |
/** * Read the last segments info from the commit pointed to by the searcher manager */ protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManager sm, final Store store) throws IOException { IndexSearcher searcher = sm.acquire(); try { IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit(); return Lucene.readSegmentInfos(latestCommit); } catch (IOException e) { // Fall back to reading from the store if reading from the commit fails try { return store. readLastCommittedSegmentsInfo(); } catch (IOException e2) { e2.addSuppressed(e); throw e2; } } finally { sm.release(searcher); } }
Example #12
Source File: TestDirectoryTaxonomyWriter.java From lucene-solr with Apache License 2.0 | 6 votes |
@Test public void testCommitNoEmptyCommits() throws Exception { // LUCENE-4972: DTW used to create empty commits even if no changes were made Directory dir = newDirectory(); DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir); taxoWriter.addCategory(new FacetLabel("a")); taxoWriter.commit(); long gen1 = SegmentInfos.getLastCommitGeneration(dir); taxoWriter.commit(); long gen2 = SegmentInfos.getLastCommitGeneration(dir); assertEquals("empty commit should not have changed the index", gen1, gen2); taxoWriter.close(); dir.close(); }
Example #13
Source File: test.java From vscode-extension with MIT License | 6 votes |
private static LocalCheckpointTracker createLocalCheckpointTracker(EngineConfig engineConfig, SegmentInfos lastCommittedSegmentInfos, Logger logger, Supplier<Searcher> searcherSupplier, BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier) { try { final SequenceNumbers.CommitInfo seqNoStats = SequenceNumbers.loadSeqNoInfoFromLuceneCommit(lastCommittedSegmentInfos.userData.entrySet()); final long maxSeqNo = seqNoStats.maxSeqNo; final long localCheckpoint = seqNoStats.localCheckpoint; logger.trace("recovered maximum sequence number [{}] and local checkpoint [{}]", maxSeqNo, localCheckpoint); final LocalCheckpointTracker tracker = localCheckpointTrackerSupplier.apply(maxSeqNo, localCheckpoint); // Operations that are optimized using max_seq_no_of_updates optimization must not be processed twice; otherwise, they will // create duplicates in Lucene. To avoid this we check the LocalCheckpointTracker to see if an operation was already processed. // Thus, we need to restore the LocalCheckpointTracker bit by bit to ensure the consistency between LocalCheckpointTracker and // Lucene index. This is not the only solution since we can bootstrap max_seq_no_of_updates with max_seq_no of the commit to // disable the MSU optimization during recovery. Here we prefer to maintain the consistency of LocalCheckpointTracker. if (localCheckpoint < maxSeqNo && engineConfig.getIndexSettings().isSoftDeleteEnabled()) { try (Searcher searcher = searcherSupplier.get()) { Lucene.scanSeqNosInReader(searcher.getDirectoryReader(), localCheckpoint + 1, maxSeqNo, tracker::markSeqNoAsCompleted); } } return tracker; } catch (IOException ex) { throw new EngineCreationFailureException(engineConfig.getShardId(), "failed to create local checkpoint tracker", ex); } }
Example #14
Source File: CopyState.java From lucene-solr with Apache License 2.0 | 5 votes |
public CopyState(Map<String,FileMetaData> files, long version, long gen, byte[] infosBytes, Set<String> completedMergeFiles, long primaryGen, SegmentInfos infos) { assert completedMergeFiles != null; this.files = Collections.unmodifiableMap(files); this.version = version; this.gen = gen; this.infosBytes = infosBytes; this.completedMergeFiles = Collections.unmodifiableSet(completedMergeFiles); this.primaryGen = primaryGen; this.infos = infos; }
Example #15
Source File: IndexFetcher.java From lucene-solr with Apache License 2.0 | 5 votes |
private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException { String segmentsFileName = commit.getSegmentsFileName(); SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName); Set<String> currentFiles = new HashSet<>(infos.files(true)); String[] allFiles = indexDir.listAll(); for (String file : allFiles) { if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) { log.info("Found unused file: {}", file); return true; } } return false; }
Example #16
Source File: SegmentsInfoRequestHandlerTest.java From lucene-solr with Apache License 2.0 | 5 votes |
@BeforeClass public static void beforeClass() throws Exception { // we need a consistent segmentation to ensure we don't get a random // merge that reduces the total num docs in all segments, or the number of deletes // systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName()); // Also prevent flushes System.setProperty("solr.tests.maxBufferedDocs", "1000"); System.setProperty("solr.tests.ramBufferSizeMB", "5000"); System.setProperty("enable.update.log", "false"); // no _version_ in our schema initCore("solrconfig.xml", "schema12.xml"); // segments API shouldn't depend on _version_ or ulog // build up an index with at least 2 segments and some deletes for (int i = 0; i < DOC_COUNT; i++) { assertU(adoc("id","SOLR100" + i, "name","Apache Solr:" + i)); } for (int i = 0; i < DEL_COUNT; i++) { assertU(delI("SOLR100" + i)); } assertU(commit()); for (int i = 0; i < DOC_COUNT; i++) { assertU(adoc("id","SOLR200" + i, "name","Apache Solr:" + i)); } assertU(commit()); h.getCore().withSearcher((searcher) -> { int numSegments = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory()).size(); // if this is not NUM_SEGMENTS, there was some unexpected flush or merge assertEquals("Unexpected number of segment in the index: " + numSegments, NUM_SEGMENTS, numSegments); return null; }); // see SOLR-14431 RefCounted<IndexWriter> iwRef = h.getCore().getSolrCoreState().getIndexWriter(h.getCore()); initialRefCount = iwRef.getRefcount(); iwRef.decref(); }
Example #17
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
private CommitPoint(SegmentInfos infos, Directory dir) throws IOException { segmentsFileName = infos.getSegmentsFileName(); this.dir = dir; userData = infos.getUserData(); files = Collections.unmodifiableCollection(infos.files(true)); generation = infos.getGeneration(); segmentCount = infos.size(); }
Example #18
Source File: TestSimpleTextCodec.java From lucene-solr with Apache License 2.0 | 5 votes |
public void test() throws Exception { SolrConfig config = h.getCore().getSolrConfig(); String codecFactory = config.get("codecFactory/@class"); assertEquals("Unexpected solrconfig codec factory", "solr.SimpleTextCodecFactory", codecFactory); assertEquals("Unexpected core codec", "SimpleText", h.getCore().getCodec().getName()); RefCounted<IndexWriter> writerRef = h.getCore().getSolrCoreState().getIndexWriter(h.getCore()); try { IndexWriter writer = writerRef.get(); assertEquals("Unexpected codec in IndexWriter config", "SimpleText", writer.getConfig().getCodec().getName()); } finally { writerRef.decref(); } assertU(add(doc("id","1", "text","textual content goes here"))); assertU(commit()); h.getCore().withSearcher(searcher -> { SegmentInfos infos = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory()); SegmentInfo info = infos.info(infos.size() - 1).info; assertEquals("Unexpected segment codec", "SimpleText", info.getCodec().getName()); return null; }); assertQ(req("q", "id:1"), "*[count(//doc)=1]"); }
Example #19
Source File: TestCodecSupport.java From lucene-solr with Apache License 2.0 | 5 votes |
protected void assertCompressionMode(String expectedModeString, SolrCore core) throws IOException { h.getCore().withSearcher(searcher -> { SegmentInfos infos = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory()); SegmentInfo info = infos.info(infos.size() - 1).info; assertEquals("Expecting compression mode string to be " + expectedModeString + " but got: " + info.getAttribute(Lucene50StoredFieldsFormat.MODE_KEY) + "\n SegmentInfo: " + info + "\n SegmentInfos: " + infos + "\n Codec: " + core.getCodec(), expectedModeString, info.getAttribute(Lucene50StoredFieldsFormat.MODE_KEY)); return null; }); }
Example #20
Source File: MergeSortRowIdMatcher.java From incubator-retired-blur with Apache License 2.0 | 5 votes |
private Collection<SegmentKey> getSegmentKeys() throws IOException { List<SegmentKey> keys = new ArrayList<SegmentKey>(); SegmentInfos segmentInfos = new SegmentInfos(); segmentInfos.read(_directory, _indexCommit.getSegmentsFileName()); for (SegmentInfoPerCommit segmentInfoPerCommit : segmentInfos) { String name = segmentInfoPerCommit.info.name; String id = getId(segmentInfoPerCommit.info); keys.add(new SegmentKey(name, id)); } return keys; }
Example #21
Source File: CommitStats.java From crate with Apache License 2.0 | 5 votes |
public CommitStats(SegmentInfos segmentInfos) { // clone the map to protect against concurrent changes userData = Map.copyOf(segmentInfos.getUserData()); // lucene calls the current generation, last generation. generation = segmentInfos.getLastGeneration(); id = Base64.getEncoder().encodeToString(segmentInfos.getId()); numDocs = Lucene.getNumDocs(segmentInfos); }
Example #22
Source File: StoreRecovery.java From crate with Apache License 2.0 | 5 votes |
private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException { final Directory directory = store.directory(); for (String name : Lucene.files(si)) { long length = directory.fileLength(name); index.addFileDetail(name, length, true); } }
Example #23
Source File: Store.java From crate with Apache License 2.0 | 5 votes |
/** * Returns the last committed segments info for this store * * @throws IOException if the index is corrupted or the segments file is not present */ public SegmentInfos readLastCommittedSegmentsInfo() throws IOException { failIfCorrupted(); try { return readSegmentsInfo(null, directory()); } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) { markStoreCorrupted(ex); throw ex; } }
Example #24
Source File: Store.java From crate with Apache License 2.0 | 5 votes |
/** * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code> * * @throws IOException if the index is corrupted or the segments file is not present */ private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException { assert commit == null || commit.getDirectory() == directory; try { return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit); } catch (EOFException eof) { // TODO this should be caught by lucene - EOF is almost certainly an index corruption throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof); } catch (IOException exception) { throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up } catch (Exception ex) { throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "commit(" + commit + ")", ex); } }
Example #25
Source File: Store.java From crate with Apache License 2.0 | 5 votes |
/** * Tries to open an index for the given location. This includes reading the * segment infos and possible corruption markers. If the index can not * be opened, an exception is thrown */ public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException { try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5)); Directory dir = new SimpleFSDirectory(indexLocation)) { failIfCorrupted(dir, shardId); SegmentInfos segInfo = Lucene.readSegmentInfos(dir); logger.trace("{} loaded segment info [{}]", shardId, segInfo); } }
Example #26
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
/** * Returns an iterable that allows to iterate over all files in this segments info */ public static Iterable<String> files(SegmentInfos infos) throws IOException { final List<Collection<String>> list = new ArrayList<>(); list.add(Collections.singleton(infos.getSegmentsFileName())); for (SegmentCommitInfo info : infos) { list.add(info.files()); } return Iterables.flatten(list); }
Example #27
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
/** * Returns the number of documents in the index referenced by this {@link SegmentInfos} */ public static int getNumDocs(SegmentInfos info) { int numDocs = 0; for (SegmentCommitInfo si : info) { numDocs += si.info.maxDoc() - si.getDelCount() - si.getSoftDelCount(); } return numDocs; }
Example #28
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
/** * Reads the segments infos from the given commit, failing if it fails to load */ public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException { // Using commit.getSegmentsFileName() does NOT work here, have to // manually create the segment filename String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration()); return SegmentInfos.readCommit(commit.getDirectory(), filename); }
Example #29
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
/** * This method removes all files from the given directory that are not referenced by the given segments file. * This method will open an IndexWriter and relies on index file deleter to remove all unreferenced files. Segment files * that are newer than the given segments file are removed forcefully to prevent problems with IndexWriter opening a potentially * broken commit point / leftover. * <b>Note:</b> this method will fail if there is another IndexWriter open on the given directory. This method will also acquire * a write lock from the directory while pruning unused files. This method expects an existing index in the given directory that has * the given segments file. */ public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Directory directory) throws IOException { final SegmentInfos si = readSegmentInfos(segmentsFileName, directory); try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) { int foundSegmentFiles = 0; for (final String file : directory.listAll()) { /** * we could also use a deletion policy here but in the case of snapshot and restore * sometimes we restore an index and override files that were referenced by a "future" * commit. If such a commit is opened by the IW it would likely throw a corrupted index exception * since checksums don's match anymore. that's why we prune the name here directly. * We also want the caller to know if we were not able to remove a segments_N file. */ if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) { foundSegmentFiles++; if (file.equals(si.getSegmentsFileName()) == false) { directory.deleteFile(file); // remove all segment_N files except of the one we wanna keep } } } assert SegmentInfos.getLastCommitSegmentsFileName(directory).equals(segmentsFileName); if (foundSegmentFiles == 0) { throw new IllegalStateException("no commit found in the directory"); } } final CommitPoint cp = new CommitPoint(si, directory); try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER) .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD) .setIndexCommit(cp) .setCommitOnClose(false) .setMergePolicy(NoMergePolicy.INSTANCE) .setOpenMode(IndexWriterConfig.OpenMode.APPEND))) { // do nothing and close this will kick of IndexFileDeleter which will remove all pending files } return si; }
Example #30
Source File: Lucene.java From crate with Apache License 2.0 | 5 votes |
public static void checkSegmentInfoIntegrity(final Directory directory) throws IOException { new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws IOException { try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) { CodecUtil.checksumEntireFile(input); } return null; } }.run(); }