org.apache.lucene.index.SegmentInfos Java Examples

The following examples show how to use org.apache.lucene.index.SegmentInfos. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CommitsImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Map<String, String> getSegmentDiagnostics(long commitGen, String name) throws LukeException {
  try {
    SegmentInfos infos = findSegmentInfos(commitGen);
    if (infos == null) {
      return Collections.emptyMap();
    }

    return infos.asList().stream()
        .filter(seg -> seg.info.name.equals(name))
        .findAny()
        .map(seg -> seg.info.getDiagnostics())
        .orElse(Collections.emptyMap());
  } catch (IOException e) {
    throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e);
  }
}
 
Example #2
Source File: SegmentsInfoRequestHandlerTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testFieldInfo() throws Exception {
  String[] segmentNamePatterns = new String[NUM_SEGMENTS];
  h.getCore().withSearcher((searcher) -> {
    int i = 0;
    for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) {
      assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS);
      segmentNamePatterns[i] = "boolean(//lst[@name='segments']/lst[@name='" + sInfo.info.name + "']/lst[@name='fields']/lst[@name='id']/str[@name='flags'])";
      i++;
    }

    return null;
  });
  assertQ("Unexpected field infos returned",
      req("qt","/admin/segments", "fieldInfo", "true"),
      segmentNamePatterns);
}
 
Example #3
Source File: SegmentsInfoRequestHandlerTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testSegmentNames() throws IOException {
  String[] segmentNamePatterns = new String[NUM_SEGMENTS];
  h.getCore().withSearcher((searcher) -> {
    int i = 0;
    for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) {
      assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS);
      segmentNamePatterns[i] = "//lst[@name='segments']/lst/str[@name='name'][.='" + sInfo.info.name + "']";
      i++;
    }
    
    return null;
  });
  assertQ("Unexpected segment names returned",
      req("qt","/admin/segments"),
      segmentNamePatterns);
}
 
Example #4
Source File: SegmentsInfoRequestHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException {
  SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
  RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
  try {
    IndexWriter indexWriter = refCounted.get();
    if (indexWriter instanceof SolrIndexWriter) {
      result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges());
    }
    //get chosen merge policy
    MergePolicy mp = indexWriter.getConfig().getMergePolicy();
    //Find merges
    MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
    if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
      for (OneMerge merge : findMerges.merges) {
        //TODO: add merge grouping
        for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
          mergeCandidates.add(mergeSegmentInfo.info.name);
        }
      }
    }

    return result;
  } finally {
    refCounted.decref();
  }
}
 
Example #5
Source File: TestSearcherTaxonomyManager.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testExceptionDuringRefresh() throws Exception {

    Directory indexDir = newDirectory();
    Directory taxoDir = newDirectory();

    IndexWriter w = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
    DirectoryTaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
    w.commit();
    tw.commit();

    SearcherTaxonomyManager mgr = new SearcherTaxonomyManager(indexDir, taxoDir, null);

    tw.addCategory(new FacetLabel("a", "b"));
    w.addDocument(new Document());

    tw.commit();
    w.commit();

    // intentionally corrupt the taxo index:
    SegmentInfos infos = SegmentInfos.readLatestCommit(taxoDir);
    taxoDir.deleteFile(infos.getSegmentsFileName());
    expectThrows(IndexNotFoundException.class, mgr::maybeRefreshBlocking);
    IOUtils.close(w, tw, mgr, indexDir, taxoDir);
  }
 
Example #6
Source File: CommitsImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<Segment> getSegments(long commitGen) throws LukeException {
  try {
    SegmentInfos infos = findSegmentInfos(commitGen);
    if (infos == null) {
      return Collections.emptyList();
    }

    return infos.asList().stream()
        .map(Segment::of)
        .sorted(Comparator.comparing(Segment::getName))
        .collect(Collectors.toList());
  } catch (IOException e) {
    throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e);
  }
}
 
Example #7
Source File: CommitsImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Map<String, String> getSegmentAttributes(long commitGen, String name) throws LukeException {
  try {
    SegmentInfos infos = findSegmentInfos(commitGen);
    if (infos == null) {
      return Collections.emptyMap();
    }

    return infos.asList().stream()
        .filter(seg -> seg.info.name.equals(name))
        .findAny()
        .map(seg -> seg.info.getAttributes())
        .orElse(Collections.emptyMap());
  } catch (IOException e) {
    throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e);
  }
}
 
Example #8
Source File: CommitsImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Optional<Codec> getSegmentCodec(long commitGen, String name) throws LukeException {
  try {
    SegmentInfos infos = findSegmentInfos(commitGen);
    if (infos == null) {
      return Optional.empty();
    }

    return infos.asList().stream()
        .filter(seg -> seg.info.name.equals(name))
        .findAny()
        .map(seg -> seg.info.getCodec());
  } catch (IOException e) {
    throw new LukeException(String.format(Locale.ENGLISH, "Failed to load segment infos for commit generation %d", commitGen), e);
  }
}
 
Example #9
Source File: TestDirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testPrepareCommitNoEmptyCommits() throws Exception {
  // LUCENE-4972: DTW used to create empty commits even if no changes were made
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.prepareCommit();
  taxoWriter.commit();
  
  long gen1 = SegmentInfos.getLastCommitGeneration(dir);
  taxoWriter.prepareCommit();
  taxoWriter.commit();
  long gen2 = SegmentInfos.getLastCommitGeneration(dir);
  assertEquals("empty commit should not have changed the index", gen1, gen2);
  
  taxoWriter.close();
  dir.close();
}
 
Example #10
Source File: TestDirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testCloseNoEmptyCommits() throws Exception {
  // LUCENE-4972: DTW used to create empty commits even if no changes were made
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.commit();
  
  long gen1 = SegmentInfos.getLastCommitGeneration(dir);
  taxoWriter.close();
  long gen2 = SegmentInfos.getLastCommitGeneration(dir);
  assertEquals("empty commit should not have changed the index", gen1, gen2);
  
  taxoWriter.close();
  dir.close();
}
 
Example #11
Source File: Engine.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
/**
 * Read the last segments info from the commit pointed to by the searcher manager
 */
protected static SegmentInfos readLastCommittedSegmentInfos(final SearcherManager sm, final Store store) throws IOException {
    IndexSearcher searcher = sm.acquire();
    try {
        IndexCommit latestCommit = ((DirectoryReader) searcher.getIndexReader()).getIndexCommit();
        return Lucene.readSegmentInfos(latestCommit);
    } catch (IOException e) {
        // Fall back to reading from the store if reading from the commit fails
        try {
            return store. readLastCommittedSegmentsInfo();
        } catch (IOException e2) {
            e2.addSuppressed(e);
            throw e2;
        }
    } finally {
        sm.release(searcher);
    }
}
 
Example #12
Source File: TestDirectoryTaxonomyWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testCommitNoEmptyCommits() throws Exception {
  // LUCENE-4972: DTW used to create empty commits even if no changes were made
  Directory dir = newDirectory();
  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(dir);
  taxoWriter.addCategory(new FacetLabel("a"));
  taxoWriter.commit();
  
  long gen1 = SegmentInfos.getLastCommitGeneration(dir);
  taxoWriter.commit();
  long gen2 = SegmentInfos.getLastCommitGeneration(dir);
  assertEquals("empty commit should not have changed the index", gen1, gen2);
  
  taxoWriter.close();
  dir.close();
}
 
Example #13
Source File: test.java    From vscode-extension with MIT License 6 votes vote down vote up
private static LocalCheckpointTracker createLocalCheckpointTracker(EngineConfig engineConfig, SegmentInfos lastCommittedSegmentInfos,
    Logger logger, Supplier<Searcher> searcherSupplier, BiFunction<Long, Long, LocalCheckpointTracker> localCheckpointTrackerSupplier) {
    try {
        final SequenceNumbers.CommitInfo seqNoStats =
            SequenceNumbers.loadSeqNoInfoFromLuceneCommit(lastCommittedSegmentInfos.userData.entrySet());
        final long maxSeqNo = seqNoStats.maxSeqNo;
        final long localCheckpoint = seqNoStats.localCheckpoint;
        logger.trace("recovered maximum sequence number [{}] and local checkpoint [{}]", maxSeqNo, localCheckpoint);
        final LocalCheckpointTracker tracker = localCheckpointTrackerSupplier.apply(maxSeqNo, localCheckpoint);
        // Operations that are optimized using max_seq_no_of_updates optimization must not be processed twice; otherwise, they will
        // create duplicates in Lucene. To avoid this we check the LocalCheckpointTracker to see if an operation was already processed.
        // Thus, we need to restore the LocalCheckpointTracker bit by bit to ensure the consistency between LocalCheckpointTracker and
        // Lucene index. This is not the only solution since we can bootstrap max_seq_no_of_updates with max_seq_no of the commit to
        // disable the MSU optimization during recovery. Here we prefer to maintain the consistency of LocalCheckpointTracker.
        if (localCheckpoint < maxSeqNo && engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
            try (Searcher searcher = searcherSupplier.get()) {
                Lucene.scanSeqNosInReader(searcher.getDirectoryReader(), localCheckpoint + 1, maxSeqNo,
                    tracker::markSeqNoAsCompleted);
            }
        }
        return tracker;
    } catch (IOException ex) {
        throw new EngineCreationFailureException(engineConfig.getShardId(), "failed to create local checkpoint tracker", ex);
    }
}
 
Example #14
Source File: CopyState.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public CopyState(Map<String,FileMetaData> files, long version, long gen, byte[] infosBytes,
                 Set<String> completedMergeFiles, long primaryGen, SegmentInfos infos) {
  assert completedMergeFiles != null;
  this.files = Collections.unmodifiableMap(files);
  this.version = version;
  this.gen = gen;
  this.infosBytes = infosBytes;
  this.completedMergeFiles = Collections.unmodifiableSet(completedMergeFiles);
  this.primaryGen = primaryGen;
  this.infos = infos;
}
 
Example #15
Source File: IndexFetcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private boolean hasUnusedFiles(Directory indexDir, IndexCommit commit) throws IOException {
  String segmentsFileName = commit.getSegmentsFileName();
  SegmentInfos infos = SegmentInfos.readCommit(indexDir, segmentsFileName);
  Set<String> currentFiles = new HashSet<>(infos.files(true));
  String[] allFiles = indexDir.listAll();
  for (String file : allFiles) {
    if (!file.equals(segmentsFileName) && !currentFiles.contains(file) && !file.endsWith(".lock")) {
      log.info("Found unused file: {}", file);
      return true;
    }
  }
  return false;
}
 
Example #16
Source File: SegmentsInfoRequestHandlerTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {

  // we need a consistent segmentation to ensure we don't get a random
  // merge that reduces the total num docs in all segments, or the number of deletes
  //
  systemSetPropertySolrTestsMergePolicyFactory(NoMergePolicyFactory.class.getName());
  // Also prevent flushes
  System.setProperty("solr.tests.maxBufferedDocs", "1000");
  System.setProperty("solr.tests.ramBufferSizeMB", "5000");
  
  System.setProperty("enable.update.log", "false"); // no _version_ in our schema
  initCore("solrconfig.xml", "schema12.xml"); // segments API shouldn't depend on _version_ or ulog
  
  // build up an index with at least 2 segments and some deletes
  for (int i = 0; i < DOC_COUNT; i++) {
    assertU(adoc("id","SOLR100" + i, "name","Apache Solr:" + i));
  }
  for (int i = 0; i < DEL_COUNT; i++) {
    assertU(delI("SOLR100" + i));
  }
  assertU(commit());
  for (int i = 0; i < DOC_COUNT; i++) {
    assertU(adoc("id","SOLR200" + i, "name","Apache Solr:" + i));
  }
  assertU(commit());
  h.getCore().withSearcher((searcher) -> {
    int numSegments = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory()).size();
    // if this is not NUM_SEGMENTS, there was some unexpected flush or merge
    assertEquals("Unexpected number of segment in the index: " + numSegments, 
        NUM_SEGMENTS, numSegments);
    return null;
  });
  // see SOLR-14431
  RefCounted<IndexWriter> iwRef = h.getCore().getSolrCoreState().getIndexWriter(h.getCore());
  initialRefCount = iwRef.getRefcount();
  iwRef.decref();
}
 
Example #17
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
private CommitPoint(SegmentInfos infos, Directory dir) throws IOException {
    segmentsFileName = infos.getSegmentsFileName();
    this.dir = dir;
    userData = infos.getUserData();
    files = Collections.unmodifiableCollection(infos.files(true));
    generation = infos.getGeneration();
    segmentCount = infos.size();
}
 
Example #18
Source File: TestSimpleTextCodec.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void test() throws Exception {
  SolrConfig config = h.getCore().getSolrConfig();
  String codecFactory =  config.get("codecFactory/@class");
  assertEquals("Unexpected solrconfig codec factory", "solr.SimpleTextCodecFactory", codecFactory);

  assertEquals("Unexpected core codec", "SimpleText", h.getCore().getCodec().getName());

  RefCounted<IndexWriter> writerRef = h.getCore().getSolrCoreState().getIndexWriter(h.getCore());
  try {
    IndexWriter writer = writerRef.get();
    assertEquals("Unexpected codec in IndexWriter config", 
        "SimpleText", writer.getConfig().getCodec().getName()); 
  } finally {
    writerRef.decref();
  }

  assertU(add(doc("id","1", "text","textual content goes here")));
  assertU(commit());

  h.getCore().withSearcher(searcher -> {
    SegmentInfos infos = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory());
    SegmentInfo info = infos.info(infos.size() - 1).info;
    assertEquals("Unexpected segment codec", "SimpleText", info.getCodec().getName());
    return null;
  });

  assertQ(req("q", "id:1"),
      "*[count(//doc)=1]");
}
 
Example #19
Source File: TestCodecSupport.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected void assertCompressionMode(String expectedModeString, SolrCore core) throws IOException {
  h.getCore().withSearcher(searcher -> {
    SegmentInfos infos = SegmentInfos.readLatestCommit(searcher.getIndexReader().directory());
    SegmentInfo info = infos.info(infos.size() - 1).info;
    assertEquals("Expecting compression mode string to be " + expectedModeString +
            " but got: " + info.getAttribute(Lucene50StoredFieldsFormat.MODE_KEY) +
            "\n SegmentInfo: " + info +
            "\n SegmentInfos: " + infos +
            "\n Codec: " + core.getCodec(),
        expectedModeString, info.getAttribute(Lucene50StoredFieldsFormat.MODE_KEY));
    return null;
  });
}
 
Example #20
Source File: MergeSortRowIdMatcher.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private Collection<SegmentKey> getSegmentKeys() throws IOException {
  List<SegmentKey> keys = new ArrayList<SegmentKey>();
  SegmentInfos segmentInfos = new SegmentInfos();
  segmentInfos.read(_directory, _indexCommit.getSegmentsFileName());
  for (SegmentInfoPerCommit segmentInfoPerCommit : segmentInfos) {
    String name = segmentInfoPerCommit.info.name;
    String id = getId(segmentInfoPerCommit.info);
    keys.add(new SegmentKey(name, id));
  }
  return keys;
}
 
Example #21
Source File: CommitStats.java    From crate with Apache License 2.0 5 votes vote down vote up
public CommitStats(SegmentInfos segmentInfos) {
    // clone the map to protect against concurrent changes
    userData = Map.copyOf(segmentInfos.getUserData());
    // lucene calls the current generation, last generation.
    generation = segmentInfos.getLastGeneration();
    id = Base64.getEncoder().encodeToString(segmentInfos.getId());
    numDocs = Lucene.getNumDocs(segmentInfos);
}
 
Example #22
Source File: StoreRecovery.java    From crate with Apache License 2.0 5 votes vote down vote up
private void addRecoveredFileDetails(SegmentInfos si, Store store, RecoveryState.Index index) throws IOException {
    final Directory directory = store.directory();
    for (String name : Lucene.files(si)) {
        long length = directory.fileLength(name);
        index.addFileDetail(name, length, true);
    }
}
 
Example #23
Source File: Store.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the last committed segments info for this store
 *
 * @throws IOException if the index is corrupted or the segments file is not present
 */
public SegmentInfos readLastCommittedSegmentsInfo() throws IOException {
    failIfCorrupted();
    try {
        return readSegmentsInfo(null, directory());
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
        markStoreCorrupted(ex);
        throw ex;
    }
}
 
Example #24
Source File: Store.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the segments info for the given commit or for the latest commit if the given commit is <code>null</code>
 *
 * @throws IOException if the index is corrupted or the segments file is not present
 */
private static SegmentInfos readSegmentsInfo(IndexCommit commit, Directory directory) throws IOException {
    assert commit == null || commit.getDirectory() == directory;
    try {
        return commit == null ? Lucene.readSegmentInfos(directory) : Lucene.readSegmentInfos(commit);
    } catch (EOFException eof) {
        // TODO this should be caught by lucene - EOF is almost certainly an index corruption
        throw new CorruptIndexException("Read past EOF while reading segment infos", "commit(" + commit + ")", eof);
    } catch (IOException exception) {
        throw exception; // IOExceptions like too many open files are not necessarily a corruption - just bubble it up
    } catch (Exception ex) {
        throw new CorruptIndexException("Hit unexpected exception while reading segment infos", "commit(" + commit + ")", ex);
    }

}
 
Example #25
Source File: Store.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Tries to open an index for the given location. This includes reading the
 * segment infos and possible corruption markers. If the index can not
 * be opened, an exception is thrown
 */
public static void tryOpenIndex(Path indexLocation, ShardId shardId, NodeEnvironment.ShardLocker shardLocker, Logger logger) throws IOException, ShardLockObtainFailedException {
    try (ShardLock lock = shardLocker.lock(shardId, "open index", TimeUnit.SECONDS.toMillis(5));
         Directory dir = new SimpleFSDirectory(indexLocation)) {
        failIfCorrupted(dir, shardId);
        SegmentInfos segInfo = Lucene.readSegmentInfos(dir);
        logger.trace("{} loaded segment info [{}]", shardId, segInfo);
    }
}
 
Example #26
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns an iterable that allows to iterate over all files in this segments info
 */
public static Iterable<String> files(SegmentInfos infos) throws IOException {
    final List<Collection<String>> list = new ArrayList<>();
    list.add(Collections.singleton(infos.getSegmentsFileName()));
    for (SegmentCommitInfo info : infos) {
        list.add(info.files());
    }
    return Iterables.flatten(list);
}
 
Example #27
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the number of documents in the index referenced by this {@link SegmentInfos}
 */
public static int getNumDocs(SegmentInfos info) {
    int numDocs = 0;
    for (SegmentCommitInfo si : info) {
        numDocs += si.info.maxDoc() - si.getDelCount() - si.getSoftDelCount();
    }
    return numDocs;
}
 
Example #28
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Reads the segments infos from the given commit, failing if it fails to load
 */
public static SegmentInfos readSegmentInfos(IndexCommit commit) throws IOException {
    // Using commit.getSegmentsFileName() does NOT work here, have to
    // manually create the segment filename
    String filename = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", commit.getGeneration());
    return SegmentInfos.readCommit(commit.getDirectory(), filename);
}
 
Example #29
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * This method removes all files from the given directory that are not referenced by the given segments file.
 * This method will open an IndexWriter and relies on index file deleter to remove all unreferenced files. Segment files
 * that are newer than the given segments file are removed forcefully to prevent problems with IndexWriter opening a potentially
 * broken commit point / leftover.
 * <b>Note:</b> this method will fail if there is another IndexWriter open on the given directory. This method will also acquire
 * a write lock from the directory while pruning unused files. This method expects an existing index in the given directory that has
 * the given segments file.
 */
public static SegmentInfos pruneUnreferencedFiles(String segmentsFileName, Directory directory) throws IOException {
    final SegmentInfos si = readSegmentInfos(segmentsFileName, directory);
    try (Lock writeLock = directory.obtainLock(IndexWriter.WRITE_LOCK_NAME)) {
        int foundSegmentFiles = 0;
        for (final String file : directory.listAll()) {
            /**
             * we could also use a deletion policy here but in the case of snapshot and restore
             * sometimes we restore an index and override files that were referenced by a "future"
             * commit. If such a commit is opened by the IW it would likely throw a corrupted index exception
             * since checksums don's match anymore. that's why we prune the name here directly.
             * We also want the caller to know if we were not able to remove a segments_N file.
             */
            if (file.startsWith(IndexFileNames.SEGMENTS) || file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
                foundSegmentFiles++;
                if (file.equals(si.getSegmentsFileName()) == false) {
                    directory.deleteFile(file); // remove all segment_N files except of the one we wanna keep
                }
            }
        }
        assert SegmentInfos.getLastCommitSegmentsFileName(directory).equals(segmentsFileName);
        if (foundSegmentFiles == 0) {
            throw new IllegalStateException("no commit found in the directory");
        }
    }
    final CommitPoint cp = new CommitPoint(si, directory);
    try (IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(Lucene.STANDARD_ANALYZER)
            .setSoftDeletesField(Lucene.SOFT_DELETES_FIELD)
            .setIndexCommit(cp)
            .setCommitOnClose(false)
            .setMergePolicy(NoMergePolicy.INSTANCE)
            .setOpenMode(IndexWriterConfig.OpenMode.APPEND))) {
        // do nothing and close this will kick of IndexFileDeleter which will remove all pending files
    }
    return si;
}
 
Example #30
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
public static void checkSegmentInfoIntegrity(final Directory directory) throws IOException {
    new SegmentInfos.FindSegmentsFile(directory) {

        @Override
        protected Object doBody(String segmentFileName) throws IOException {
            try (IndexInput input = directory.openInput(segmentFileName, IOContext.READ)) {
                CodecUtil.checksumEntireFile(input);
            }
            return null;
        }
    }.run();
}