Java Code Examples for org.apache.lucene.codecs.CodecUtil#writeIndexHeader()

public FSTTermsWriter(SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException {
  final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);

  this.postingsWriter = postingsWriter;
  this.fieldInfos = state.fieldInfos;
  this.out = state.directory.createOutput(termsFileName, state.context);
  this.maxDoc = state.segmentInfo.maxDoc();

  boolean success = false;
  try {
    CodecUtil.writeIndexHeader(out, TERMS_CODEC_NAME, TERMS_VERSION_CURRENT,
                                      state.segmentInfo.getId(), state.segmentSuffix);   

    this.postingsWriter.init(out, state); 
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 

public FixedGapTermsIndexWriter(SegmentWriteState state, int termIndexInterval) throws IOException {
  if (termIndexInterval <= 0) {
    throw new IllegalArgumentException("invalid termIndexInterval: " + termIndexInterval);
  }
  this.termIndexInterval = termIndexInterval;
  final String indexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
  out = state.directory.createOutput(indexFileName, state.context);
  boolean success = false;
  try {
    CodecUtil.writeIndexHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    out.writeVInt(termIndexInterval);
    out.writeVInt(PackedInts.VERSION_CURRENT);
    out.writeVInt(BLOCKSIZE);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 

public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter,
    SegmentWriteState state, PostingsWriterBase postingsWriter)
    throws IOException {
  final String termsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_EXTENSION);
  this.termsIndexWriter = termsIndexWriter;
  maxDoc = state.segmentInfo.maxDoc();
  out = state.directory.createOutput(termsFileName, state.context);
  boolean success = false;
  try {
    fieldInfos = state.fieldInfos;
    CodecUtil.writeIndexHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    currentField = null;
    this.postingsWriter = postingsWriter;
    // segment = state.segmentName;
    
    //System.out.println("BTW.init seg=" + state.segmentName);
    
    postingsWriter.init(out, state); // have consumer write its format/header
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 

public void testCorruptFilesAreCaught() throws Exception {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";

  // wrong checksum
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i=0; i < 1024; i++) {
      os.writeByte((byte) i);
    }

    // write footer w/ wrong checksum
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);

    long checksum = os.getChecksum();
    os.writeLong(checksum+1);
  }

  si.setFiles(Collections.singletonList(subFile));
  Exception e = expectThrows(CorruptIndexException.class, () -> si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT));
  assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
  dir.close();
}
 

CompletionFieldsConsumer(String codecName, PostingsFormat delegatePostingsFormat, SegmentWriteState state) throws IOException {
  this.codecName = codecName;
  this.delegatePostingsFormatName = delegatePostingsFormat.getName();
  this.state = state;
  String dictFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, DICT_EXTENSION);
  boolean success = false;
  try {
    this.delegateFieldsConsumer = delegatePostingsFormat.fieldsConsumer(state);
    dictOut = state.directory.createOutput(dictFile, state.context);
    CodecUtil.writeIndexHeader(dictOut, codecName, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(dictOut, delegateFieldsConsumer);
    }
  }
}
 

/** expert: Creates a new writer */
public Lucene80DocValuesConsumer(SegmentWriteState state, String dataCodec, String dataExtension, String metaCodec, String metaExtension) throws IOException {
  boolean success = false;
  try {
    this.state = state;
    String dataName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dataExtension);
    data = state.directory.createOutput(dataName, state.context);
    CodecUtil.writeIndexHeader(data, dataCodec, Lucene80DocValuesFormat.VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    String metaName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, metaExtension);
    meta = state.directory.createOutput(metaName, state.context);
    CodecUtil.writeIndexHeader(meta, metaCodec, Lucene80DocValuesFormat.VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    maxDoc = state.segmentInfo.maxDoc();
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(this);
    }
  }
}
 

public void testDoubleClose() throws IOException {
  final String testfile = "_123.test";

  Directory dir = newDirectory();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput out = dir.createOutput(testfile, IOContext.DEFAULT)) {
    CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
    out.writeInt(3);
    CodecUtil.writeFooter(out);
  }
  
  si.setFiles(Collections.singleton(testfile));
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  assertEquals(1, cfs.listAll().length);
  cfs.close();
  cfs.close(); // second close should not throw exception
  dir.close();
}
 

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
    }
    CodecUtil.writeFooter(output);
  }
}
 

public void testCheckIntegrity() throws IOException {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i = 0; i < 1024; i++) {
      os.writeByte((byte) i);
    }
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);
    long checksum = os.getChecksum();
    os.writeLong(checksum);
  }

  si.setFiles(Collections.singletonList(subFile));
  
  FileTrackingDirectoryWrapper writeTrackingDir = new FileTrackingDirectoryWrapper(dir);
  si.getCodec().compoundFormat().write(writeTrackingDir, si, IOContext.DEFAULT);
  final Set<String> createdFiles = writeTrackingDir.getFiles();

  ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
  CompoundDirectory compoundDir = si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.READ);
  compoundDir.checkIntegrity();
  Map<String,FixedBitSet> readBytes = readTrackingDir.getReadBytes();
  assertEquals(createdFiles, readBytes.keySet());
  for (Map.Entry<String, FixedBitSet> entry : readBytes.entrySet()) {
    final String file = entry.getKey();
    final FixedBitSet set = entry.getValue().clone();
    set.flip(0, set.length());
    final int next = set.nextSetBit(0);
    assertEquals("Byte at offset " + next + " of " + file + " was not read", DocIdSetIterator.NO_MORE_DOCS, next);
  }
  compoundDir.close();
  dir.close();
}
 

/** Creates a file of the specified size with random data. */
protected static void createRandomFile(Directory dir, String name, int size, byte[] segId) throws IOException {
  Random rnd = random();
  try (IndexOutput os = dir.createOutput(name, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, segId, "suffix");
    for (int i=0; i<size; i++) {
      byte b = (byte) rnd.nextInt(256);
      os.writeByte(b);
    }
    CodecUtil.writeFooter(os);
  }
}
 

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
      output.writeVInt(fi.getPointDimensionCount());
      if (fi.getPointDimensionCount() != 0) {
        output.writeVInt(fi.getPointIndexDimensionCount());
        output.writeVInt(fi.getPointNumBytes());
      }
    }
    CodecUtil.writeFooter(output);
  }
}
 

@Override
public void finish() throws IOException {
  if (finished) {
    throw new IllegalStateException("already finished");
  }
  finished = true;
  CodecUtil.writeFooter(dataOut);

  String indexFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name,
                                                        writeState.segmentSuffix,
                                                        Lucene60PointsFormat.INDEX_EXTENSION);
  // Write index file
  try (IndexOutput indexOut = writeState.directory.createOutput(indexFileName, writeState.context)) {
    CodecUtil.writeIndexHeader(indexOut,
                               Lucene60PointsFormat.META_CODEC_NAME,
                               Lucene60PointsFormat.INDEX_VERSION_CURRENT,
                               writeState.segmentInfo.getId(),
                               writeState.segmentSuffix);
    int count = indexFPs.size();
    indexOut.writeVInt(count);
    for(Map.Entry<String,Long> ent : indexFPs.entrySet()) {
      FieldInfo fieldInfo = writeState.fieldInfos.fieldInfo(ent.getKey());
      if (fieldInfo == null) {
        throw new IllegalStateException("wrote field=\"" + ent.getKey() + "\" but that field doesn't exist in FieldInfos");
      }
      indexOut.writeVInt(fieldInfo.number);
      indexOut.writeVLong(ent.getValue());
    }
    CodecUtil.writeFooter(indexOut);
  }
}
 

@Override
public void close() throws IOException {
  if (closed) {
    return;
  }
  closed = true;
  String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
  boolean success = false;
  try (IndexOutput indexOut = state.directory.createOutput(indexFile, state.context)) {
    delegateFieldsConsumer.close();
    CodecUtil.writeIndexHeader(indexOut, codecName, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    /*
     * we write the delegate postings format name so we can load it
     * without getting an instance in the ctor
     */
    indexOut.writeString(delegatePostingsFormatName);
    // write # of seen fields
    indexOut.writeVInt(seenFields.size());
    // write field numbers and dictOut offsets
    for (Map.Entry<String, CompletionMetaData> seenField : seenFields.entrySet()) {
      FieldInfo fieldInfo = state.fieldInfos.fieldInfo(seenField.getKey());
      indexOut.writeVInt(fieldInfo.number);
      CompletionMetaData metaData = seenField.getValue();
      indexOut.writeVLong(metaData.filePointer);
      indexOut.writeVLong(metaData.minWeight);
      indexOut.writeVLong(metaData.maxWeight);
      indexOut.writeByte(metaData.type);
    }
    CodecUtil.writeFooter(indexOut);
    CodecUtil.writeFooter(dictOut);
    IOUtils.close(dictOut);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(dictOut, delegateFieldsConsumer);
    }
  }
}
 

/**
 * @param targetNumBlockLines Target number of lines per block.
 *                            Must be strictly greater than 0.
 *                            The parameters can be pre-validated with {@link #validateSettings(int, int)}.
 *                            There is one term per block line, with its corresponding details ({@link org.apache.lucene.index.TermState}).
 * @param deltaNumLines       Maximum allowed delta variation of the number of lines per block.
 *                            Must be greater than or equal to 0 and strictly less than {@code targetNumBlockLines}.
 *                            The block size will be {@code targetNumBlockLines}+-{@code deltaNumLines}.
 *                            The block size must always be less than or equal to {@link #MAX_NUM_BLOCK_LINES}.
 * @param blockEncoder        Optional block encoder, may be null if none.
 *                            It can be used for compression or encryption.
 */
protected UniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                        int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder, FieldMetadata.Serializer fieldMetadataWriter,
                        String codecName, int versionCurrent, String termsBlocksExtension, String dictionaryExtension) throws IOException {
  validateSettings(targetNumBlockLines, deltaNumLines);
  IndexOutput blockOutput = null;
  IndexOutput dictionaryOutput = null;
  boolean success = false;
  try {
    this.fieldInfos = state.fieldInfos;
    this.postingsWriter = postingsWriter;
    this.maxDoc = state.segmentInfo.maxDoc();
    this.targetNumBlockLines = targetNumBlockLines;
    this.deltaNumLines = deltaNumLines;
    this.blockEncoder = blockEncoder;
    this.fieldMetadataWriter = fieldMetadataWriter;

    String termsName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, termsBlocksExtension);
    blockOutput = state.directory.createOutput(termsName, state.context);
    CodecUtil.writeIndexHeader(blockOutput, codecName, versionCurrent, state.segmentInfo.getId(), state.segmentSuffix);

    String indexName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, dictionaryExtension);
    dictionaryOutput = state.directory.createOutput(indexName, state.context);
    CodecUtil.writeIndexHeader(dictionaryOutput, codecName, versionCurrent, state.segmentInfo.getId(), state.segmentSuffix);

    postingsWriter.init(blockOutput, state);

    this.blockOutput = blockOutput;
    this.dictionaryOutput = dictionaryOutput;
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(blockOutput, dictionaryOutput);
    }
  }
}
 

public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException {
  final String indexFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, TERMS_INDEX_EXTENSION);
  out = state.directory.createOutput(indexFileName, state.context);
  boolean success = false;
  try {
    fieldInfos = state.fieldInfos;
    this.policy = policy;
    CodecUtil.writeIndexHeader(out, CODEC_NAME, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    }
  }
}
 

@Override
public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
  CodecUtil.writeIndexHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
}
 

public void testManySubFiles() throws IOException {
  final MockDirectoryWrapper dir = newMockFSDirectory(createTempDir("CFSManySubFiles"));
  
  final int FILE_COUNT = atLeast(500);
  
  List<String> files = new ArrayList<>();
  SegmentInfo si = newSegmentInfo(dir, "_123");
  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    String file = "_123." + fileIdx;
    files.add(file);
    try (IndexOutput out = dir.createOutput(file, newIOContext(random()))) {
      CodecUtil.writeIndexHeader(out, "Foo", 0, si.getId(), "suffix");
      out.writeByte((byte) fileIdx);
      CodecUtil.writeFooter(out);
    }
  }
  
  assertEquals(0, dir.getFileHandleCount());
  
  si.setFiles(files);
  si.getCodec().compoundFormat().write(dir, si, IOContext.DEFAULT);
  Directory cfs = si.getCodec().compoundFormat().getCompoundReader(dir, si, IOContext.DEFAULT);
  
  final IndexInput[] ins = new IndexInput[FILE_COUNT];
  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    ins[fileIdx] = cfs.openInput("_123." + fileIdx, newIOContext(random()));
    CodecUtil.checkIndexHeader(ins[fileIdx], "Foo", 0, 0, si.getId(), "suffix");
  }
  
  assertEquals(1, dir.getFileHandleCount());

  for (int fileIdx = 0; fileIdx < FILE_COUNT; fileIdx++) {
    assertEquals((byte) fileIdx, ins[fileIdx].readByte());
  }
  
  assertEquals(1, dir.getFileHandleCount());
  
  for(int fileIdx=0;fileIdx<FILE_COUNT;fileIdx++) {
    ins[fileIdx].close();
  }
  cfs.close();
  
  dir.close();
}
 

@Override
public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
  CodecUtil.writeIndexHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
  termsOut.writeVInt(BLOCK_SIZE);
}
 

@Override
public void init(IndexOutput termsOut, SegmentWriteState state) throws IOException {
  CodecUtil.writeIndexHeader(termsOut, TERMS_CODEC, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
  termsOut.writeVInt(BLOCK_SIZE);
}
 

/** Creates a postings writer */
public Lucene50PostingsWriter(SegmentWriteState state) throws IOException {
  final float acceptableOverheadRatio = PackedInts.COMPACT;

  String docFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.DOC_EXTENSION);
  docOut = state.directory.createOutput(docFileName, state.context);
  IndexOutput posOut = null;
  IndexOutput payOut = null;
  boolean success = false;
  try {
    CodecUtil.writeIndexHeader(docOut, DOC_CODEC, VERSION_CURRENT, 
                                 state.segmentInfo.getId(), state.segmentSuffix);
    forUtil = new ForUtil(acceptableOverheadRatio, docOut);
    if (state.fieldInfos.hasProx()) {
      posDeltaBuffer = new int[MAX_DATA_SIZE];
      String posFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.POS_EXTENSION);
      posOut = state.directory.createOutput(posFileName, state.context);
      CodecUtil.writeIndexHeader(posOut, POS_CODEC, VERSION_CURRENT,
                                   state.segmentInfo.getId(), state.segmentSuffix);

      if (state.fieldInfos.hasPayloads()) {
        payloadBytes = new byte[128];
        payloadLengthBuffer = new int[MAX_DATA_SIZE];
      } else {
        payloadBytes = null;
        payloadLengthBuffer = null;
      }

      if (state.fieldInfos.hasOffsets()) {
        offsetStartDeltaBuffer = new int[MAX_DATA_SIZE];
        offsetLengthBuffer = new int[MAX_DATA_SIZE];
      } else {
        offsetStartDeltaBuffer = null;
        offsetLengthBuffer = null;
      }

      if (state.fieldInfos.hasPayloads() || state.fieldInfos.hasOffsets()) {
        String payFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, Lucene50PostingsFormat.PAY_EXTENSION);
        payOut = state.directory.createOutput(payFileName, state.context);
        CodecUtil.writeIndexHeader(payOut, PAY_CODEC, VERSION_CURRENT,
                                     state.segmentInfo.getId(), state.segmentSuffix);
      }
    } else {
      posDeltaBuffer = null;
      payloadLengthBuffer = null;
      offsetStartDeltaBuffer = null;
      offsetLengthBuffer = null;
      payloadBytes = null;
    }
    this.payOut = payOut;
    this.posOut = posOut;
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(docOut, posOut, payOut);
    }
  }

  docDeltaBuffer = new int[MAX_DATA_SIZE];
  freqBuffer = new int[MAX_DATA_SIZE];

  // TODO: should we try skipping every 2/4 blocks...?
  skipWriter = new Lucene50SkipWriter(MAX_SKIP_LEVELS,
                                      BLOCK_SIZE, 
                                      state.segmentInfo.maxDoc(),
                                      docOut,
                                      posOut,
                                      payOut);

  encoded = new byte[MAX_ENCODED_SIZE];
}