org.apache.lucene.codecs.FieldsConsumer Java Examples

The following examples show how to use org.apache.lucene.codecs.FieldsConsumer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: STUniformSplitRot13PostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      recordBlockEncodingCall();
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 
Example #2
Source File: Lucene50RWPostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, 
                                                  BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #3
Source File: UniformSplitRot13PostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 
Example #4
Source File: BlockTreeOrdsPostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new OrdsBlockTreeTermsWriter(state, 
                                                      postingsWriter,
                                                      minTermBlockSize, 
                                                      maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #5
Source File: Lucene84PostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  minTermBlockSize, 
                                                  maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #6
Source File: IDVersionPostingsFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new IDVersionPostingsWriter(state.liveDocs);
  boolean success = false;
  try {
    FieldsConsumer ret = new VersionBlockTreeTermsWriter(state, 
                                                         postingsWriter,
                                                         minTermsInBlock, 
                                                         maxTermsInBlock);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #7
Source File: CompletionPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsFormat delegatePostingsFormat = delegatePostingsFormat();
  if (delegatePostingsFormat == null) {
    throw new UnsupportedOperationException("Error - " + getClass().getName()
        + " has been constructed without a choice of PostingsFormat");
  }
  return new CompletionFieldsConsumer(getName(), delegatePostingsFormat, state);
}
 
Example #8
Source File: CrankyPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from PostingsFormat.fieldsConsumer()");
  }  
  return new CrankyFieldsConsumer(delegate.fieldsConsumer(state), random);
}
 
Example #9
Source File: LuceneVarGapFixedInterval.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNTermSelector(termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example #10
Source File: LuceneFixedGap.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example #11
Source File: LuceneVarGapDocFreqInterval.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThreshold, termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example #12
Source File: RAMOnlyPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException {
  final int id = nextID.getAndIncrement();

  // TODO -- ok to do this up front instead of
  // on close....?  should be ok?
  // Write our ID:
  final String idFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, ID_EXTENSION);
  IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST);
    out.writeVInt(id);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    } else {
      IOUtils.close(out);
    }
  }
  
  final RAMPostings postings = new RAMPostings();
  final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);

  synchronized(state) {
    state.put(id, postings);
  }
  return consumer;
}
 
Example #13
Source File: PerFieldPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void write(Fields fields, NormsProducer norms) throws IOException {
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(fields);

  // Write postings
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      // Exposes only the fields from this group:
      Fields maskedFields = new FilterFields(fields) {
        @Override
        public Iterator<String> iterator() {
          return group.fields.iterator();
        }
      };

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.write(maskedFields, norms);
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 
Example #14
Source File: PerFieldPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void merge(MergeState mergeState, NormsProducer norms) throws IOException {
  @SuppressWarnings("unchecked") Iterable<String> indexedFieldNames = () ->
      new MergedIterator<>(true,
          Arrays.stream(mergeState.fieldsProducers).map(FieldsProducer::iterator).toArray(Iterator[]::new));
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(indexedFieldNames);

  // Merge postings
  PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.merge(pfMergeState.apply(group.fields), norms);
    }
    success = true;
  } finally {
    pfMergeState.reset();
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 
Example #15
Source File: UniformSplitPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer termsWriter = createUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
    success = true;
    return termsWriter;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #16
Source File: FSTPostingsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example #17
Source File: MtasCodecPostingsFormat.java    From mtas with Apache License 2.0 5 votes vote down vote up
@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  if (delegatePostingsFormat != null) {
    return new MtasFieldsConsumer(
        delegatePostingsFormat.fieldsConsumer(state), state, getName(),
        delegatePostingsFormat.getName());
  } else {
    PostingsFormat pf = Codec.forName(delegateCodecName).postingsFormat();
    return pf.fieldsConsumer(state);
  }
}
 
Example #18
Source File: MtasFieldsConsumer.java    From mtas with Apache License 2.0 4 votes vote down vote up
/**
 * Instantiates a new mtas fields consumer.
 *
 * @param fieldsConsumer
 *          the fields consumer
 * @param state
 *          the state
 * @param name
 *          the name
 * @param delegatePostingsFormatName
 *          the delegate postings format name
 */
public MtasFieldsConsumer(FieldsConsumer fieldsConsumer,
    SegmentWriteState state, String name, String delegatePostingsFormatName) {
  this.delegateFieldsConsumer = fieldsConsumer;
  this.state = state;
  this.name = name;
  this.delegatePostingsFormatName = delegatePostingsFormatName;
  // temporary fileNames
  mtasTmpFieldFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_FIELD_EXTENSION);
  mtasTmpObjectFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_OBJECT_EXTENSION);
  mtasTmpDocsFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOCS_EXTENSION);
  mtasTmpDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TMP_DOC_EXTENSION);
  mtasTmpDocsChainedFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_TMP_DOCS_CHAINED_EXTENSION);
  // fileNames
  mtasObjectFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_OBJECT_EXTENSION);
  mtasTermFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_TERM_EXTENSION);
  mtasIndexFieldFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_FIELD_EXTENSION);
  mtasPrefixFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_PREFIX_EXTENSION);
  mtasDocFileName = IndexFileNames.segmentFileName(state.segmentInfo.name,
      state.segmentSuffix, MtasCodecPostingsFormat.MTAS_DOC_EXTENSION);
  mtasIndexDocIdFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_DOC_ID_EXTENSION);
  mtasIndexObjectIdFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_ID_EXTENSION);
  mtasIndexObjectPositionFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_POSITION_EXTENSION);
  mtasIndexObjectParentFileName = IndexFileNames.segmentFileName(
      state.segmentInfo.name, state.segmentSuffix,
      MtasCodecPostingsFormat.MTAS_INDEX_OBJECT_PARENT_EXTENSION);
}
 
Example #19
Source File: CrankyPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
CrankyFieldsConsumer(FieldsConsumer delegate, Random random) {
  this.delegate = delegate;
  this.random = random;
}
 
Example #20
Source File: PerFieldPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return new FieldsWriter(state);
}
 
Example #21
Source File: SegmentMerger.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
private void mergeTerms(SegmentWriteState segmentWriteState, NormsProducer norms) throws IOException {
  try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) {
    consumer.merge(mergeState, norms);
  }
}
 
Example #22
Source File: Lucene50PostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  throw new UnsupportedOperationException("Old formats can't be used for writing");
}
 
Example #23
Source File: TestBloomFilteredLucenePostings.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return delegate.fieldsConsumer(state);
}
 
Example #24
Source File: AssertingPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
AssertingFieldsConsumer(SegmentWriteState writeState, FieldsConsumer in) {
  this.writeState = writeState;
  this.in = in;
}
 
Example #25
Source File: AssertingPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new AssertingFieldsConsumer(state, in.fieldsConsumer(state));
}
 
Example #26
Source File: STUniformSplitPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 
Example #27
Source File: UniformSplitPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 
Example #28
Source File: BloomFilteringPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
    SegmentWriteState state) {
  this.delegateFieldsConsumer = fieldsConsumer;
  this.state = state;
}
 
Example #29
Source File: SimpleTextPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new SimpleTextFieldsWriter(state);
}
 
Example #30
Source File: DirectPostingsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return PostingsFormat.forName("Lucene84").fieldsConsumer(state);
}