Java Code Examples for org.apache.lucene.codecs.Codec#getDefault()

@Override
public PostingsFormat postingsFormat() {
  initDelegate();
  if (delegate.postingsFormat() instanceof PerFieldPostingsFormat) {
    Codec defaultCodec = Codec.getDefault();
    PostingsFormat defaultPostingsFormat = defaultCodec.postingsFormat();
    if (defaultPostingsFormat instanceof PerFieldPostingsFormat) {
      defaultPostingsFormat = ((PerFieldPostingsFormat) defaultPostingsFormat)
          .getPostingsFormatForField(null);
      if ((defaultPostingsFormat == null)
          || (defaultPostingsFormat instanceof PerFieldPostingsFormat)) {
        // fallback option
        return new MtasCodecPostingsFormat(
            PostingsFormat.forName("Lucene70"));
      } else {
        return new MtasCodecPostingsFormat(defaultPostingsFormat);
      }
    } else {
      return new MtasCodecPostingsFormat(defaultPostingsFormat);
    }
  } else {
    return new MtasCodecPostingsFormat(delegate.postingsFormat());
  }
}
 

public void testIsFullyDeleted() throws IOException {
  Directory dir = new ByteBuffersDirectory();
  SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 3, false, Codec.getDefault(),
      Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
  FieldInfos fieldInfos = FieldInfos.EMPTY;
  si.getCodec().fieldInfosFormat().write(dir, si, "", fieldInfos, IOContext.DEFAULT);
  PendingDeletes deletes = newPendingDeletes(commitInfo);
  for (int i = 0; i < 3; i++) {
    assertTrue(deletes.delete(i));
    if (random().nextBoolean()) {
      assertTrue(deletes.writeLiveDocs(dir));
    }
    assertEquals(i == 2, deletes.isFullyDeleted(() -> null));
  }
}
 

LiveIndexWriterConfig(Analyzer analyzer) {
  this.analyzer = analyzer;
  ramBufferSizeMB = IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB;
  maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS;
  mergedSegmentWarmer = null;
  delPolicy = new KeepOnlyLastCommitDeletionPolicy();
  commit = null;
  useCompoundFile = IndexWriterConfig.DEFAULT_USE_COMPOUND_FILE_SYSTEM;
  openMode = OpenMode.CREATE_OR_APPEND;
  similarity = IndexSearcher.getDefaultSimilarity();
  mergeScheduler = new ConcurrentMergeScheduler();
  indexingChain = DocumentsWriterPerThread.defaultIndexingChain;
  codec = Codec.getDefault();
  if (codec == null) {
    throw new NullPointerException();
  }
  infoStream = InfoStream.getDefault();
  mergePolicy = new TieredMergePolicy();
  flushPolicy = new FlushByRamOrCountsPolicy();
  readerPooling = IndexWriterConfig.DEFAULT_READER_POOLING;
  perThreadHardLimitMB = IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB;
  maxCommitMergeWaitMillis = IndexWriterConfig.DEFAULT_MAX_COMMIT_MERGE_WAIT_MILLIS;
}
 

public void testVersionsOneSegment() throws IOException {
  BaseDirectoryWrapper dir = newDirectory();
  dir.setCheckIndexOnClose(false);
  byte id[] = StringHelper.randomId();
  Codec codec = Codec.getDefault();

  SegmentInfos sis = new SegmentInfos(Version.LATEST.major);
  SegmentInfo info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_0", 1, false, Codec.getDefault(),
                                     Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
  info.setFiles(Collections.<String>emptySet());
  codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());

  sis.add(commitInfo);
  sis.commit(dir);
  sis = SegmentInfos.readLatestCommit(dir);
  assertEquals(Version.LUCENE_9_0_0, sis.getMinSegmentLuceneVersion());
  assertEquals(Version.LATEST, sis.getCommitLuceneVersion());
  dir.close();
}
 

private SegmentCommitInfo merge(Directory dir, SegmentCommitInfo si1, SegmentCommitInfo si2, String merged, boolean useCompoundFile)
  throws Exception {
  IOContext context = newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1)));
  SegmentReader r1 = new SegmentReader(si1, Version.LATEST.major, context);
  SegmentReader r2 = new SegmentReader(si2, Version.LATEST.major, context);

  final Codec codec = Codec.getDefault();
  TrackingDirectoryWrapper trackingDir = new TrackingDirectoryWrapper(si1.info.dir);
  final SegmentInfo si = new SegmentInfo(si1.info.dir, Version.LATEST, null, merged, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);

  SegmentMerger merger = new SegmentMerger(Arrays.<CodecReader>asList(r1, r2),
                                           si, InfoStream.getDefault(), trackingDir,
                                           new FieldInfos.FieldNumbers(null), context);

  MergeState mergeState = merger.merge();
  r1.close();
  r2.close();;
  si.setFiles(new HashSet<>(trackingDir.getCreatedFiles()));
    
  if (useCompoundFile) {
    Collection<String> filesToDelete = si.files();
    codec.compoundFormat().write(dir, si, context);
    si.setUseCompoundFile(true);
    for(String name : filesToDelete) {
      si1.info.dir.deleteFile(name);
    }
  }

  return new SegmentCommitInfo(si, 0, 0, -1L, -1L, -1L, StringHelper.randomId());
}
 

private static Codec getCodec() {
  if (Codec.getDefault().getName().equals("Lucene84")) {
    int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048);
    double maxMBSortInHeap = 5.0 + (3*random().nextDouble());
    if (VERBOSE) {
      System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
    }

    return new FilterCodec("Lucene84", Codec.getDefault()) {
      @Override
      public PointsFormat pointsFormat() {
        return new PointsFormat() {
          @Override
          public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
            return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
          }

          @Override
          public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
            return new Lucene86PointsReader(readState);
          }
        };
      }
    };
  } else {
    return Codec.getDefault();
  }
}
 

public void testVersionsTwoSegments() throws IOException {
  BaseDirectoryWrapper dir = newDirectory();
  dir.setCheckIndexOnClose(false);
  byte id[] = StringHelper.randomId();
  Codec codec = Codec.getDefault();

  SegmentInfos sis = new SegmentInfos(Version.LATEST.major);
  SegmentInfo info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_0", 1, false, Codec.getDefault(),
                                     Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
  info.setFiles(Collections.<String>emptySet());
  codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());
  sis.add(commitInfo);

  info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_1", 1, false, Codec.getDefault(),
                         Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
  info.setFiles(Collections.<String>emptySet());
  codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
  commitInfo = new SegmentCommitInfo(info, 0, 0,-1, -1, -1, StringHelper.randomId());
  sis.add(commitInfo);

  sis.commit(dir);
  byte[] commitInfoId0 = sis.info(0).getId();
  byte[] commitInfoId1 = sis.info(1).getId();
  sis = SegmentInfos.readLatestCommit(dir);
  assertEquals(Version.LUCENE_9_0_0, sis.getMinSegmentLuceneVersion());
  assertEquals(Version.LATEST, sis.getCommitLuceneVersion());
  assertEquals(StringHelper.idToString(commitInfoId0), StringHelper.idToString(sis.info(0).getId()));
  assertEquals(StringHelper.idToString(commitInfoId1), StringHelper.idToString(sis.info(1).getId()));
  dir.close();
}
 

private static Codec getCodec() {
  if (Codec.getDefault().getName().equals("Lucene84")) {
    int maxPointsInLeafNode = TestUtil.nextInt(random(), 16, 2048);
    double maxMBSortInHeap = 3.0 + (3*random().nextDouble());
    if (VERBOSE) {
      System.out.println("TEST: using Lucene60PointsFormat with maxPointsInLeafNode=" + maxPointsInLeafNode + " and maxMBSortInHeap=" + maxMBSortInHeap);
    }

    return new FilterCodec("Lucene84", Codec.getDefault()) {
      @Override
      public PointsFormat pointsFormat() {
        return new PointsFormat() {
          @Override
          public PointsWriter fieldsWriter(SegmentWriteState writeState) throws IOException {
            return new Lucene86PointsWriter(writeState, maxPointsInLeafNode, maxMBSortInHeap);
          }

          @Override
          public PointsReader fieldsReader(SegmentReadState readState) throws IOException {
            return new Lucene86PointsReader(readState);
          }
        };
      }
    };
  } else {
    return Codec.getDefault();
  }
}
 

public void testIDChangesOnAdvance() throws IOException {
  try (BaseDirectoryWrapper dir = newDirectory()) {
    dir.setCheckIndexOnClose(false);
    byte id[] = StringHelper.randomId();
    SegmentInfo info = new SegmentInfo(dir, Version.LUCENE_9_0_0, Version.LUCENE_9_0_0, "_0", 1, false, Codec.getDefault(),
        Collections.<String, String>emptyMap(), StringHelper.randomId(), Collections.<String, String>emptyMap(), null);
    SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, id);
    assertEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
    commitInfo.advanceDelGen();
    assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));

    id = commitInfo.getId();
    commitInfo.advanceDocValuesGen();
    assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));

    id = commitInfo.getId();
    commitInfo.advanceFieldInfosGen();
    assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
    SegmentCommitInfo clone = commitInfo.clone();
    id = commitInfo.getId();
    assertEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
    assertEquals(StringHelper.idToString(id), StringHelper.idToString(clone.getId()));

    commitInfo.advanceFieldInfosGen();
    assertNotEquals(StringHelper.idToString(id), StringHelper.idToString(commitInfo.getId()));
    assertEquals("clone changed but shouldn't", StringHelper.idToString(id), StringHelper.idToString(clone.getId()));
  }
}
 

public void testDeleteDoc() throws IOException {
  Directory dir = new ByteBuffersDirectory();
  SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 10, false, Codec.getDefault(),
      Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0, -1, -1, -1, StringHelper.randomId());
  PendingDeletes deletes = newPendingDeletes(commitInfo);
  assertNull(deletes.getLiveDocs());
  int docToDelete = TestUtil.nextInt(random(), 0, 7);
  assertTrue(deletes.delete(docToDelete));
  assertNotNull(deletes.getLiveDocs());
  assertEquals(1, deletes.numPendingDeletes());

  Bits liveDocs = deletes.getLiveDocs();
  assertFalse(liveDocs.get(docToDelete));
  assertFalse(deletes.delete(docToDelete)); // delete again

  assertTrue(liveDocs.get(8));
  assertTrue(deletes.delete(8));
  assertTrue(liveDocs.get(8)); // we have a snapshot
  assertEquals(2, deletes.numPendingDeletes());

  assertTrue(liveDocs.get(9));
  assertTrue(deletes.delete(9));
  assertTrue(liveDocs.get(9));

  // now make sure new live docs see the deletions
  liveDocs = deletes.getLiveDocs();
  assertFalse(liveDocs.get(9));
  assertFalse(liveDocs.get(8));
  assertFalse(liveDocs.get(docToDelete));
  assertEquals(3, deletes.numPendingDeletes());
  dir.close();
}
 

/** Test toString method */
public void testToString() throws Throwable{
  SegmentInfo si;
  final Directory dir = newDirectory();
  Codec codec = Codec.getDefault();

  // diagnostics map
  Map<String, String> diagnostics = Map.of("key1", "value1", "key2", "value2");

  // attributes map
  Map<String,String> attributes =  Map.of("akey1", "value1", "akey2", "value2");

  // diagnostics X, attributes X
  si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "TEST", 10000, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), Sort.INDEXORDER);
  assertEquals("TEST(" + Version.LATEST.toString() + ")" +
      ":C10000" +
      ":[indexSort=<doc>]", si.toString());

  // diagnostics O, attributes X
  si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "TEST", 10000, false, codec, diagnostics, StringHelper.randomId(), new HashMap<>(), Sort.INDEXORDER);
  assertEquals("TEST(" + Version.LATEST.toString() + ")" +
      ":C10000" +
      ":[indexSort=<doc>]" +
      ":[diagnostics=" + diagnostics + "]", si.toString());

  // diagnostics X, attributes O
  si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "TEST", 10000, false, codec, Collections.emptyMap(), StringHelper.randomId(), attributes, Sort.INDEXORDER);
  assertEquals("TEST(" + Version.LATEST.toString() + ")" +
      ":C10000" +
      ":[indexSort=<doc>]" +
      ":[attributes=" + attributes + "]", si.toString());

  // diagnostics O, attributes O
  si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "TEST", 10000, false, codec, diagnostics, StringHelper.randomId(), attributes, Sort.INDEXORDER);
  assertEquals("TEST(" + Version.LATEST.toString() + ")" +
      ":C10000" +
      ":[indexSort=<doc>]" +
      ":[diagnostics=" + diagnostics + "]" +
      ":[attributes=" + attributes + "]", si.toString());

  dir.close();
}
 

/**
 * Inits the delegate.
 */
private void initDelegate() {
  if (delegate == null) {
    delegate = Codec.getDefault();
  }
}
 

public void testMerge() throws IOException {
  final Codec codec = Codec.getDefault();
  final SegmentInfo si = new SegmentInfo(mergedDir, Version.LATEST, null, mergedSegment, -1, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);

  SegmentMerger merger = new SegmentMerger(Arrays.<CodecReader>asList(reader1, reader2),
                                           si, InfoStream.getDefault(), mergedDir,
                                           new FieldInfos.FieldNumbers(null),
                                           newIOContext(random(), new IOContext(new MergeInfo(-1, -1, false, -1))));
  MergeState mergeState = merger.merge();
  int docsMerged = mergeState.segmentInfo.maxDoc();
  assertTrue(docsMerged == 2);
  //Should be able to open a new SegmentReader against the new directory
  SegmentReader mergedReader = new SegmentReader(new SegmentCommitInfo(
      mergeState.segmentInfo,
      0, 0, -1L, -1L, -1L, StringHelper.randomId()),
      Version.LATEST.major,
      newIOContext(random()));
  assertTrue(mergedReader != null);
  assertTrue(mergedReader.numDocs() == 2);
  Document newDoc1 = mergedReader.document(0);
  assertTrue(newDoc1 != null);
  //There are 2 unstored fields on the document
  assertTrue(DocHelper.numFields(newDoc1) == DocHelper.numFields(doc1) - DocHelper.unstored.size());
  Document newDoc2 = mergedReader.document(1);
  assertTrue(newDoc2 != null);
  assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size());

  PostingsEnum termDocs = TestUtil.docs(random(), mergedReader,
      DocHelper.TEXT_FIELD_2_KEY,
      new BytesRef("field"),
      null,
      0);
  assertTrue(termDocs != null);
  assertTrue(termDocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);

  int tvCount = 0;
  for(FieldInfo fieldInfo : mergedReader.getFieldInfos()) {
    if (fieldInfo.hasVectors()) {
      tvCount++;
    }
  }
  
  //System.out.println("stored size: " + stored.size());
  assertEquals("We do not have 3 fields that were indexed with term vector", 3, tvCount);

  Terms vector = mergedReader.getTermVectors(0).terms(DocHelper.TEXT_FIELD_2_KEY);
  assertNotNull(vector);
  assertEquals(3, vector.size());
  TermsEnum termsEnum = vector.iterator();

  int i = 0;
  while (termsEnum.next() != null) {
    String term = termsEnum.term().utf8ToString();
    int freq = (int) termsEnum.totalTermFreq();
    //System.out.println("Term: " + term + " Freq: " + freq);
    assertTrue(DocHelper.FIELD_2_TEXT.indexOf(term) != -1);
    assertTrue(DocHelper.FIELD_2_FREQS[i] == freq);
    i++;
  }

  TestSegmentReader.checkNorms(mergedReader);
  mergedReader.close();
}
 

public void testWriteLiveDocs() throws IOException {
  Directory dir = new ByteBuffersDirectory();
  SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "test", 6, false, Codec.getDefault(),
      Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(si, 0, 0,  -1, -1, -1, StringHelper.randomId());
  PendingDeletes deletes = newPendingDeletes(commitInfo);
  assertFalse(deletes.writeLiveDocs(dir));
  assertEquals(0, dir.listAll().length);
  boolean secondDocDeletes = random().nextBoolean();
  deletes.delete(5);
  if (secondDocDeletes) {
    deletes.getLiveDocs();
    deletes.delete(2);
  }
  assertEquals(-1, commitInfo.getDelGen());
  assertEquals(0, commitInfo.getDelCount());

  assertEquals(secondDocDeletes ? 2 : 1, deletes.numPendingDeletes());
  assertTrue(deletes.writeLiveDocs(dir));
  assertEquals(1, dir.listAll().length);
  Bits liveDocs = Codec.getDefault().liveDocsFormat().readLiveDocs(dir, commitInfo, IOContext.DEFAULT);
  assertFalse(liveDocs.get(5));
  if (secondDocDeletes) {
    assertFalse(liveDocs.get(2));
  } else {
    assertTrue(liveDocs.get(2));
  }
  assertTrue(liveDocs.get(0));
  assertTrue(liveDocs.get(1));
  assertTrue(liveDocs.get(3));
  assertTrue(liveDocs.get(4));

  assertEquals(0, deletes.numPendingDeletes());
  assertEquals(secondDocDeletes ? 2 : 1, commitInfo.getDelCount());
  assertEquals(1, commitInfo.getDelGen());

  deletes.delete(0);
  assertTrue(deletes.writeLiveDocs(dir));
  assertEquals(2, dir.listAll().length);
  liveDocs = Codec.getDefault().liveDocsFormat().readLiveDocs(dir, commitInfo, IOContext.DEFAULT);
  assertFalse(liveDocs.get(5));
  if (secondDocDeletes) {
    assertFalse(liveDocs.get(2));
  } else {
    assertTrue(liveDocs.get(2));
  }
  assertFalse(liveDocs.get(0));
  assertTrue(liveDocs.get(1));
  assertTrue(liveDocs.get(3));
  assertTrue(liveDocs.get(4));

  assertEquals(0, deletes.numPendingDeletes());
  assertEquals(secondDocDeletes ? 3 : 2, commitInfo.getDelCount());
  assertEquals(2, commitInfo.getDelGen());
  dir.close();
}
 

public void testFixedPostings() throws Throwable {
  final int NUM_TERMS = 100;
  final TermData[] terms = new TermData[NUM_TERMS];
  for(int i=0;i<NUM_TERMS;i++) {
    final int[] docs = new int[] {i};
    final String text = Integer.toString(i, Character.MAX_RADIX);
    terms[i] = new TermData(text, docs, null);
  }

  final FieldInfos.Builder builder = new FieldInfos.Builder(new FieldInfos.FieldNumbers(null));

  final FieldData field = new FieldData("field", builder, terms, true, false);
  final FieldData[] fields = new FieldData[] {field};
  final FieldInfos fieldInfos = builder.finish();
  final Directory dir = newDirectory();
  Codec codec = Codec.getDefault();
  final SegmentInfo si = new SegmentInfo(dir, Version.LATEST, Version.LATEST, SEGMENT, 10000, false, codec, Collections.emptyMap(), StringHelper.randomId(), new HashMap<>(), null);
  
  this.write(si, fieldInfos, dir, fields);
  final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random())));

  final Iterator<String> fieldsEnum = reader.iterator();
  String fieldName = fieldsEnum.next();
  assertNotNull(fieldName);
  final Terms terms2 = reader.terms(fieldName);
  assertNotNull(terms2);

  final TermsEnum termsEnum = terms2.iterator();

  PostingsEnum postingsEnum = null;
  for(int i=0;i<NUM_TERMS;i++) {
    final BytesRef term = termsEnum.next();
    assertNotNull(term);
    assertEquals(terms[i].text2, term.utf8ToString());

    // do this twice to stress test the codec's reuse, ie,
    // make sure it properly fully resets (rewinds) its
    // internal state:
    for(int iter=0;iter<2;iter++) {
      postingsEnum = TestUtil.docs(random(), termsEnum, postingsEnum, PostingsEnum.NONE);
      assertEquals(terms[i].docs[0], postingsEnum.nextDoc());
      assertEquals(DocIdSetIterator.NO_MORE_DOCS, postingsEnum.nextDoc());
    }
  }
  assertNull(termsEnum.next());

  for(int i=0;i<NUM_TERMS;i++) {
    assertEquals(termsEnum.seekCeil(new BytesRef(terms[i].text2)), TermsEnum.SeekStatus.FOUND);
  }

  assertFalse(fieldsEnum.hasNext());
  reader.close();
  dir.close();
}
 

public void testBitFlippedTriggersCorruptIndexException() throws IOException {
  BaseDirectoryWrapper dir = newDirectory();
  dir.setCheckIndexOnClose(false);
  byte id[] = StringHelper.randomId();
  Codec codec = Codec.getDefault();

  SegmentInfos sis = new SegmentInfos(Version.LATEST.major);
  SegmentInfo info = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_0", 1, false, Codec.getDefault(),
                                     Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
  info.setFiles(Collections.<String>emptySet());
  codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
  SegmentCommitInfo commitInfo = new SegmentCommitInfo(info, 0, 0, -1, -1, -1, StringHelper.randomId());
  sis.add(commitInfo);

  info = new SegmentInfo(dir, Version.LATEST, Version.LATEST, "_1", 1, false, Codec.getDefault(),
                         Collections.<String,String>emptyMap(), id, Collections.<String,String>emptyMap(), null);
  info.setFiles(Collections.<String>emptySet());
  codec.segmentInfoFormat().write(dir, info, IOContext.DEFAULT);
  commitInfo = new SegmentCommitInfo(info, 0, 0,-1, -1, -1, StringHelper.randomId());
  sis.add(commitInfo);

  sis.commit(dir);

  BaseDirectoryWrapper corruptDir = newDirectory();
  corruptDir.setCheckIndexOnClose(false);
  boolean corrupt = false;
  for (String file : dir.listAll()) {
    if (file.startsWith(IndexFileNames.SEGMENTS)) {
      try (IndexInput in = dir.openInput(file, IOContext.DEFAULT);
          IndexOutput out = corruptDir.createOutput(file, IOContext.DEFAULT)) {
        final long corruptIndex = TestUtil.nextLong(random(), 0, in.length() - 1);
        out.copyBytes(in, corruptIndex);
        final int b = Byte.toUnsignedInt(in.readByte()) + TestUtil.nextInt(random(), 0x01, 0xff);
        out.writeByte((byte) b);
        out.copyBytes(in, in.length() - in.getFilePointer());
      }
      try (IndexInput in = corruptDir.openInput(file, IOContext.DEFAULT)) {
        CodecUtil.checksumEntireFile(in);
        if (VERBOSE) {
          System.out.println("TEST: Altering the file did not update the checksum, aborting...");
        }
        return;
      } catch (CorruptIndexException e) {
        // ok
      }
      corrupt = true;
    } else if (ExtrasFS.isExtra(file) == false) {
      corruptDir.copyFrom(dir, file, file, IOContext.DEFAULT);
    }
  }
  assertTrue("No segments file found", corrupt);

  expectThrowsAnyOf(
      Arrays.asList(CorruptIndexException.class, IndexFormatTooOldException.class, IndexFormatTooNewException.class),
      () -> SegmentInfos.readLatestCommit(corruptDir));
  dir.close();
  corruptDir.close();
}
 

/** Returns a new fake segment */
protected static SegmentInfo newSegmentInfo(Directory dir, String name) {
  Version minVersion = random().nextBoolean() ? null : Version.LATEST;
  return new SegmentInfo(dir, Version.LATEST, minVersion,  name, 10000, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), Collections.emptyMap(), null);
}
 

/** Returns a new fake segment */
protected static SegmentInfo newSegmentInfo(Directory dir, String name) {
  Version minVersion = random().nextBoolean() ? null : Version.LATEST;
  return new SegmentInfo(dir, Version.LATEST, minVersion, name, 10000, false, Codec.getDefault(), Collections.emptyMap(), StringHelper.randomId(), Collections.emptyMap(), null);
}
 

/** Test the accuracy of the ramBytesUsed estimations. */
@Nightly
public void testRamBytesUsed() throws IOException {
  if (Codec.getDefault() instanceof RandomCodec) {
    // this test relies on the fact that two segments will be written with
    // the same codec so we need to disable MockRandomPF
    final Set<String> avoidCodecs = new HashSet<>(((RandomCodec) Codec.getDefault()).avoidCodecs);
    avoidCodecs.add(new MockRandomPostingsFormat().getName());
    Codec.setDefault(new RandomCodec(random(), avoidCodecs));
  }
  Directory dir = applyCreatedVersionMajor(newDirectory());
  IndexWriterConfig cfg = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w = new IndexWriter(dir, cfg);
  // we need to index enough documents so that constant overhead doesn't dominate
  final int numDocs = atLeast(10000);
  LeafReader reader1 = null;
  for (int i = 0; i < numDocs; ++i) {
    Document d = new Document();
    addRandomFields(d);
    w.addDocument(d);
    if (i == 100) {
      w.forceMerge(1);
      w.commit();
      reader1 = getOnlyLeafReader(DirectoryReader.open(dir));
    }
  }
  w.forceMerge(1);
  w.commit();
  w.close();

  LeafReader reader2 = getOnlyLeafReader(DirectoryReader.open(dir));

  for (LeafReader reader : Arrays.asList(reader1, reader2)) {
    new SimpleMergedSegmentWarmer(InfoStream.NO_OUTPUT).warm(reader);
  }

  long act1 = RamUsageTester.sizeOf(reader2, new Accumulator(reader2));
  long act2 = RamUsageTester.sizeOf(reader1, new Accumulator(reader1));
  final long measuredBytes = act1 - act2;

  long reported1 = ((SegmentReader) reader2).ramBytesUsed();
  long reported2 = ((SegmentReader) reader1).ramBytesUsed();
  final long reportedBytes = reported1 - reported2;

  final long absoluteError = Math.abs(measuredBytes - reportedBytes);
  final double relativeError = (double) absoluteError / measuredBytes;
  final String message = String.format(Locale.ROOT,
      "RamUsageTester reports %d bytes but ramBytesUsed() returned %d (%.1f error). " +
      " [Measured: %d, %d. Reported: %d, %d]",
      measuredBytes,
      reportedBytes,
      (100 * relativeError),
      act1, act2,
      reported1, reported2);

  assertTrue(message, relativeError < 0.20d || absoluteError < 1000);

  reader1.close();
  reader2.close();
  dir.close();
}
 

public void setUp() throws Exception {
  super.setUp();
  // set the default codec, so adding test cases to this isn't fragile
  savedCodec = Codec.getDefault();
  Codec.setDefault(getCodec());
}