org.apache.lucene.index.SegmentCommitInfo Java Examples

The following examples show how to use org.apache.lucene.index.SegmentCommitInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: DLBasedEngine.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 
Example #2
Source File: InternalEngine.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 
Example #3
Source File: test.java    From vscode-extension with MIT License 6 votes vote down vote up
@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 
Example #4
Source File: ElasticsearchMergePolicy.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private boolean shouldUpgrade(SegmentCommitInfo info) {
    org.apache.lucene.util.Version old = info.info.getVersion();
    org.apache.lucene.util.Version cur = Version.CURRENT.luceneVersion;

    // Something seriously wrong if this trips:
    assert old.major <= cur.major;

    if (cur.major > old.major) {
        // Always upgrade segment if Lucene's major version is too old
        return true;
    }
    if (upgradeOnlyAncientSegments == false && cur.minor > old.minor) {
        // If it's only a minor version difference, and we are not upgrading only ancient segments,
        // also upgrade:
        return true;
    }
    // Version matches, or segment is not ancient and we are only upgrading ancient segments:
    return false;
}
 
Example #5
Source File: ElasticsearchMergePolicy.java    From crate with Apache License 2.0 6 votes vote down vote up
private boolean shouldUpgrade(SegmentCommitInfo info) {
    org.apache.lucene.util.Version old = info.info.getVersion();
    org.apache.lucene.util.Version cur = Version.CURRENT.luceneVersion;

    // Something seriously wrong if this trips:
    assert old.major <= cur.major;

    if (cur.major > old.major) {
        // Always upgrade segment if Lucene's major version is too old
        return true;
    }
    if (upgradeOnlyAncientSegments == false && cur.minor > old.minor) {
        // If it's only a minor version difference, and we are not upgrading only ancient segments,
        // also upgrade:
        return true;
    }
    // Version matches, or segment is not ancient and we are only upgrading ancient segments:
    return false;
}
 
Example #6
Source File: InternalEngine.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public List<Segment> segments(boolean verbose) {
    try (ReleasableLock lock = readLock.acquire()) {
        Segment[] segmentsArr = getSegmentInfo(lastCommittedSegmentInfos, verbose);

        // fill in the merges flag
        Set<OnGoingMerge> onGoingMerges = mergeScheduler.onGoingMerges();
        for (OnGoingMerge onGoingMerge : onGoingMerges) {
            for (SegmentCommitInfo segmentInfoPerCommit : onGoingMerge.getMergedSegments()) {
                for (Segment segment : segmentsArr) {
                    if (segment.getName().equals(segmentInfoPerCommit.info.name)) {
                        segment.mergeId = onGoingMerge.getId();
                        break;
                    }
                }
            }
        }
        return Arrays.asList(segmentsArr);
    }
}
 
Example #7
Source File: Engine.java    From crate with Apache License 2.0 6 votes vote down vote up
protected final DocsStats docsStats(IndexReader indexReader) {
    long numDocs = 0;
    long numDeletedDocs = 0;
    long sizeInBytes = 0;
    // we don't wait for a pending refreshes here since it's a stats call instead we mark it as accessed only which will cause
    // the next scheduled refresh to go through and refresh the stats as well
    for (LeafReaderContext readerContext : indexReader.leaves()) {
        // we go on the segment level here to get accurate numbers
        final SegmentReader segmentReader = Lucene.segmentReader(readerContext.reader());
        SegmentCommitInfo info = segmentReader.getSegmentInfo();
        numDocs += readerContext.reader().numDocs();
        numDeletedDocs += readerContext.reader().numDeletedDocs();
        try {
            sizeInBytes += info.sizeInBytes();
        } catch (IOException e) {
            logger.trace(() -> new ParameterizedMessage("failed to get size for [{}]", info.info.name), e);
        }
    }
    return new DocsStats(numDocs, numDeletedDocs, sizeInBytes);
}
 
Example #8
Source File: SegmentsInfoRequestHandlerTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testFieldInfo() throws Exception {
  String[] segmentNamePatterns = new String[NUM_SEGMENTS];
  h.getCore().withSearcher((searcher) -> {
    int i = 0;
    for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) {
      assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS);
      segmentNamePatterns[i] = "boolean(//lst[@name='segments']/lst[@name='" + sInfo.info.name + "']/lst[@name='fields']/lst[@name='id']/str[@name='flags'])";
      i++;
    }

    return null;
  });
  assertQ("Unexpected field infos returned",
      req("qt","/admin/segments", "fieldInfo", "true"),
      segmentNamePatterns);
}
 
Example #9
Source File: SegmentsInfoRequestHandlerTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testSegmentNames() throws IOException {
  String[] segmentNamePatterns = new String[NUM_SEGMENTS];
  h.getCore().withSearcher((searcher) -> {
    int i = 0;
    for (SegmentCommitInfo sInfo : SegmentInfos.readLatestCommit(searcher.getIndexReader().directory())) {
      assertTrue("Unexpected number of segment in the index: " + i, i < NUM_SEGMENTS);
      segmentNamePatterns[i] = "//lst[@name='segments']/lst/str[@name='name'][.='" + sInfo.info.name + "']";
      i++;
    }
    
    return null;
  });
  assertQ("Unexpected segment names returned",
      req("qt","/admin/segments"),
      segmentNamePatterns);
}
 
Example #10
Source File: SegmentsInfoRequestHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private SimpleOrderedMap<Object> getMergeInformation(SolrQueryRequest req, SegmentInfos infos, List<String> mergeCandidates) throws IOException {
  SimpleOrderedMap<Object> result = new SimpleOrderedMap<>();
  RefCounted<IndexWriter> refCounted = req.getCore().getSolrCoreState().getIndexWriter(req.getCore());
  try {
    IndexWriter indexWriter = refCounted.get();
    if (indexWriter instanceof SolrIndexWriter) {
      result.addAll(((SolrIndexWriter)indexWriter).getRunningMerges());
    }
    //get chosen merge policy
    MergePolicy mp = indexWriter.getConfig().getMergePolicy();
    //Find merges
    MergeSpecification findMerges = mp.findMerges(MergeTrigger.EXPLICIT, infos, indexWriter);
    if (findMerges != null && findMerges.merges != null && findMerges.merges.size() > 0) {
      for (OneMerge merge : findMerges.merges) {
        //TODO: add merge grouping
        for (SegmentCommitInfo mergeSegmentInfo : merge.segments) {
          mergeCandidates.add(mergeSegmentInfo.info.name);
        }
      }
    }

    return result;
  } finally {
    refCounted.decref();
  }
}
 
Example #11
Source File: Lucene50LiveDocsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
  long gen = info.getNextDelGen();
  String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
  int delCount = 0;
  try (IndexOutput output = dir.createOutput(name, context)) {
    CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
    final int longCount = FixedBitSet.bits2words(bits.length());
    for (int i = 0; i < longCount; ++i) {
      long currentBits = 0;
      for (int j = i << 6, end = Math.min(j + 63, bits.length() - 1); j <= end; ++j) {
        if (bits.get(j)) {
          currentBits |= 1L << j; // mod 64
        } else {
          delCount += 1;
        }
      }
      output.writeLong(currentBits);
    }
    CodecUtil.writeFooter(output);
  }
  if (delCount != info.getDelCount() + newDelCount) {
    throw new CorruptIndexException("bits.deleted=" + delCount + 
        " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name);
  }
}
 
Example #12
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the number of documents in the index referenced by this {@link SegmentInfos}
 */
public static int getNumDocs(SegmentInfos info) {
    int numDocs = 0;
    for (SegmentCommitInfo si : info) {
        numDocs += si.info.maxDoc() - si.getDelCount() - si.getSoftDelCount();
    }
    return numDocs;
}
 
Example #13
Source File: Lucene.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns an iterable that allows to iterate over all files in this segments info
 */
public static Iterable<String> files(SegmentInfos infos) throws IOException {
    final List<Collection<String>> list = new ArrayList<>();
    list.add(Collections.singleton(infos.getSegmentsFileName()));
    for (SegmentCommitInfo info : infos) {
        list.add(info.files());
    }
    return Iterables.flatten(list);
}
 
Example #14
Source File: Lucene50LiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
  long gen = info.getDelGen();
  String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
  final int length = info.info.maxDoc();
  try (ChecksumIndexInput input = dir.openChecksumInput(name, context)) {
    Throwable priorE = null;
    try {
      CodecUtil.checkIndexHeader(input, CODEC_NAME, VERSION_START, VERSION_CURRENT, 
                                   info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
      long data[] = new long[FixedBitSet.bits2words(length)];
      for (int i = 0; i < data.length; i++) {
        data[i] = input.readLong();
      }
      FixedBitSet fbs = new FixedBitSet(data, length);
      if (fbs.length() - fbs.cardinality() != info.getDelCount()) {
        throw new CorruptIndexException("bits.deleted=" + (fbs.length() - fbs.cardinality()) + 
                                        " info.delcount=" + info.getDelCount(), input);
      }
      return fbs.asReadOnlyBits();
    } catch (Throwable exception) {
      priorE = exception;
    } finally {
      CodecUtil.checkFooter(input, priorE);
    }
  }
  throw new AssertionError();
}
 
Example #15
Source File: CrankyLiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from LiveDocsFormat.writeLiveDocs()");
  }
  delegate.writeLiveDocs(bits, dir, info, newDelCount, context);
}
 
Example #16
Source File: AssertingLiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
  Bits raw = in.readLiveDocs(dir, info, context);
  assert raw != null;
  check(raw, info.info.maxDoc(), info.getDelCount());
  return new AssertingBits(raw);
}
 
Example #17
Source File: SimpleTextLiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void writeLiveDocs(Bits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
  int size = bits.length();
  BytesRefBuilder scratch = new BytesRefBuilder();
  
  String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getNextDelGen());
  IndexOutput out = null;
  boolean success = false;
  try {
    out = dir.createOutput(fileName, context);
    SimpleTextUtil.write(out, SIZE);
    SimpleTextUtil.write(out, Integer.toString(size), scratch);
    SimpleTextUtil.writeNewline(out);
    
    for (int i = 0; i < size; ++i) {
      if (bits.get(i)) {
        SimpleTextUtil.write(out, DOC);
        SimpleTextUtil.write(out, Integer.toString(i), scratch);
        SimpleTextUtil.writeNewline(out);
      }
    }
    
    SimpleTextUtil.write(out, END);
    SimpleTextUtil.writeNewline(out);
    SimpleTextUtil.writeChecksum(out, scratch);
    success = true;
  } finally {
    if (success) {
      IOUtils.close(out);
    } else {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 
Example #18
Source File: SimpleTextLiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
  assert info.hasDeletions();
  BytesRefBuilder scratch = new BytesRefBuilder();
  CharsRefBuilder scratchUTF16 = new CharsRefBuilder();
  
  String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getDelGen());
  ChecksumIndexInput in = null;
  boolean success = false;
  try {
    in = dir.openChecksumInput(fileName, context);
    
    SimpleTextUtil.readLine(in, scratch);
    assert StringHelper.startsWith(scratch.get(), SIZE);
    int size = parseIntAt(scratch.get(), SIZE.length, scratchUTF16);
    
    BitSet bits = new BitSet(size);
    
    SimpleTextUtil.readLine(in, scratch);
    while (!scratch.get().equals(END)) {
      assert StringHelper.startsWith(scratch.get(), DOC);
      int docid = parseIntAt(scratch.get(), DOC.length, scratchUTF16);
      bits.set(docid);
      SimpleTextUtil.readLine(in, scratch);
    }
    
    SimpleTextUtil.checkFooter(in);
    
    success = true;
    return new SimpleTextBits(bits, size);
  } finally {
    if (success) {
      IOUtils.close(in);
    } else {
      IOUtils.closeWhileHandlingException(in);
    }
  }
}
 
Example #19
Source File: Segment.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static Segment of(SegmentCommitInfo segInfo) {
  Segment segment = new Segment();
  segment.name = segInfo.info.name;
  segment.maxDoc = segInfo.info.maxDoc();
  segment.delGen = segInfo.getDelGen();
  segment.delCount = segInfo.getDelCount();
  segment.luceneVer = segInfo.info.getVersion().toString();
  segment.codecName = segInfo.info.getCodec().getName();
  try {
    segment.displaySize = CommitsImpl.toDisplaySize(segInfo.sizeInBytes());
  } catch (IOException e) {
  }
  segment.useCompoundFile = segInfo.info.getUseCompoundFile();
  return segment;
}
 
Example #20
Source File: OnGoingMerge.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
/**
 * The list of segments that are being merged.
 */
public List<SegmentCommitInfo> getMergedSegments() {
    return mergedSegments;
}
 
Example #21
Source File: CrankyLiveDocsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
  return delegate.readLiveDocs(dir, info, context);
}
 
Example #22
Source File: ElasticsearchMergePolicy.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
public IndexUpgraderOneMerge(List<SegmentCommitInfo> segments) {
    super(segments);
}
 
Example #23
Source File: ElasticsearchMergePolicy.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
    int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, MergeContext mergeContext)
    throws IOException {

    if (upgradeInProgress) {
        MergeSpecification spec = new MergeSpecification();
        for (SegmentCommitInfo info : segmentInfos) {

            if (shouldUpgrade(info)) {

                // TODO: Use IndexUpgradeMergePolicy instead.  We should be comparing codecs,
                // for now we just assume every minor upgrade has a new format.
                logger.debug("Adding segment {} to be upgraded", info.info.name);
                spec.add(new OneMerge(Collections.singletonList(info)));
            }

            // TODO: we could check IndexWriter.getMergingSegments and avoid adding merges that IW will just reject?

            if (spec.merges.size() == MAX_CONCURRENT_UPGRADE_MERGES) {
                // hit our max upgrades, so return the spec.  we will get a cascaded call to continue.
                logger.debug("Returning {} merges for upgrade", spec.merges.size());
                return spec;
            }
        }

        // We must have less than our max upgrade merges, so the next return will be our last in upgrading mode.
        if (spec.merges.isEmpty() == false) {
            logger.debug("Returning {} merges for end of upgrade", spec.merges.size());
            return spec;
        }

        // Only set this once there are 0 segments needing upgrading, because when we return a
        // spec, IndexWriter may (silently!) reject that merge if some of the segments we asked
        // to be merged were already being (naturally) merged:
        upgradeInProgress = false;

        // fall through, so when we don't have any segments to upgrade, the delegate policy
        // has a chance to decide what to do (e.g. collapse the segments to satisfy maxSegmentCount)
    }

    return super.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, mergeContext);
}
 
Example #24
Source File: ElasticsearchMergePolicy.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos,
    int maxSegmentCount, Map<SegmentCommitInfo,Boolean> segmentsToMerge, IndexWriter writer)
    throws IOException {

    if (upgradeInProgress) {
        MergeSpecification spec = new IndexUpgraderMergeSpecification();
        for (SegmentCommitInfo info : segmentInfos) {

            if (shouldUpgrade(info)) {

                // TODO: Use IndexUpgradeMergePolicy instead.  We should be comparing codecs,
                // for now we just assume every minor upgrade has a new format.
                logger.debug("Adding segment " + info.info.name + " to be upgraded");
                spec.add(new OneMerge(Collections.singletonList(info)));
            }

            // TODO: we could check IndexWriter.getMergingSegments and avoid adding merges that IW will just reject?

            if (spec.merges.size() == MAX_CONCURRENT_UPGRADE_MERGES) {
                // hit our max upgrades, so return the spec.  we will get a cascaded call to continue.
                logger.debug("Returning " + spec.merges.size() + " merges for upgrade");
                return spec;
            }
        }

        // We must have less than our max upgrade merges, so the next return will be our last in upgrading mode.
        if (spec.merges.isEmpty() == false) {
            logger.debug("Returning " + spec.merges.size() + " merges for end of upgrade");
            return spec;
        }

        // Only set this once there are 0 segments needing upgrading, because when we return a
        // spec, IndexWriter may (silently!) reject that merge if some of the segments we asked
        // to be merged were already being (naturally) merged:
        upgradeInProgress = false;

        // fall through, so when we don't have any segments to upgrade, the delegate policy
        // has a chance to decide what to do (e.g. collapse the segments to satisfy maxSegmentCount)
    }

    return upgradedMergeSpecification(delegate.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer));
}
 
Example #25
Source File: OnGoingMerge.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * The list of segments that are being merged.
 */
public List<SegmentCommitInfo> getMergedSegments() {
    return oneMerge.segments;
}
 
Example #26
Source File: EsTieredMergePolicy.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCount,
        Map<SegmentCommitInfo, Boolean> segmentsToMerge, MergeContext mergeContext) throws IOException {
    return forcedMergePolicy.findForcedMerges(infos, maxSegmentCount, segmentsToMerge, mergeContext);
}
 
Example #27
Source File: ElasticsearchMergePolicy.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public boolean useCompoundFile(SegmentInfos segments, SegmentCommitInfo newSegment, IndexWriter writer) throws IOException {
    return delegate.useCompoundFile(segments, newSegment, writer);
}
 
Example #28
Source File: SortingMergePolicyDecorator.java    From linden with Apache License 2.0 4 votes vote down vote up
@Override
public MergeSpecification findForcedMerges(SegmentInfos segmentInfos, int maxSegmentCount,
                                           Map<SegmentCommitInfo, Boolean> segmentsToMerge,
                                           IndexWriter writer) throws IOException {
  return sortingMergePolicy.findForcedMerges(segmentInfos, maxSegmentCount, segmentsToMerge, writer);
}
 
Example #29
Source File: PrimaryNode.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Called when a merge has finished, but before IW switches to the merged segment */
protected abstract void preCopyMergedSegmentFiles(SegmentCommitInfo info, Map<String,FileMetaData> files) throws IOException;
 
Example #30
Source File: UninvertDocValuesMergePolicyFactory.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public UninvertDocValuesOneMerge(List<SegmentCommitInfo> segments) {
  super(segments);
}