Java Code Examples for org.apache.lucene.codecs.FieldsConsumer

The following examples show how to use org.apache.lucene.codecs.FieldsConsumer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: BlockTreeOrdsPostingsFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new OrdsBlockTreeTermsWriter(state, 
                                                      postingsWriter,
                                                      minTermBlockSize, 
                                                      maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 2
Source Project: lucene-solr   Source File: IDVersionPostingsFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new IDVersionPostingsWriter(state.liveDocs);
  boolean success = false;
  try {
    FieldsConsumer ret = new VersionBlockTreeTermsWriter(state, 
                                                         postingsWriter,
                                                         minTermsInBlock, 
                                                         maxTermsInBlock);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 3
protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 
Example 4
protected FieldsConsumer createFieldsConsumer(SegmentWriteState segmentWriteState, PostingsWriterBase postingsWriter) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, segmentWriteState,
      UniformSplitTermsWriter.DEFAULT_TARGET_NUM_BLOCK_LINES,
      UniformSplitTermsWriter.DEFAULT_DELTA_NUM_LINES,
      getBlockEncoder()
  ) {
    @Override
    protected void writeDictionary(IndexDictionary.Builder dictionaryBuilder) throws IOException {
      recordBlockEncodingCall();
      super.writeDictionary(dictionaryBuilder);
      recordDictionaryEncodingCall();
    }
    @Override
    protected void writeEncodedFieldsMetadata(ByteBuffersDataOutput fieldsOutput) throws IOException {
      recordBlockEncodingCall();
      super.writeEncodedFieldsMetadata(fieldsOutput);
      recordFieldsMetadataEncodingCall();
    }
  };
}
 
Example 5
Source Project: lucene-solr   Source File: Lucene84PostingsFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  minTermBlockSize, 
                                                  maxTermBlockSize);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 6
Source Project: lucene-solr   Source File: Lucene50RWPostingsFormat.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene50PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer ret = new BlockTreeTermsWriter(state, 
                                                  postingsWriter,
                                                  BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, 
                                                  BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 7
Source Project: lucene-solr   Source File: FSTPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);

  boolean success = false;
  try {
    FieldsConsumer ret = new FSTTermsWriter(state, postingsWriter);
    success = true;
    return ret;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 8
Source Project: lucene-solr   Source File: UniformSplitPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase postingsWriter = new Lucene84PostingsWriter(state);
  boolean success = false;
  try {
    FieldsConsumer termsWriter = createUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
    success = true;
    return termsWriter;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(postingsWriter);
    }
  }
}
 
Example 9
Source Project: lucene-solr   Source File: CompletionPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsFormat delegatePostingsFormat = delegatePostingsFormat();
  if (delegatePostingsFormat == null) {
    throw new UnsupportedOperationException("Error - " + getClass().getName()
        + " has been constructed without a choice of PostingsFormat");
  }
  return new CompletionFieldsConsumer(getName(), delegatePostingsFormat, state);
}
 
Example 10
Source Project: lucene-solr   Source File: RAMOnlyPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException {
  final int id = nextID.getAndIncrement();

  // TODO -- ok to do this up front instead of
  // on close....?  should be ok?
  // Write our ID:
  final String idFileName = IndexFileNames.segmentFileName(writeState.segmentInfo.name, writeState.segmentSuffix, ID_EXTENSION);
  IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context);
  boolean success = false;
  try {
    CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST);
    out.writeVInt(id);
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(out);
    } else {
      IOUtils.close(out);
    }
  }
  
  final RAMPostings postings = new RAMPostings();
  final RAMFieldsConsumer consumer = new RAMFieldsConsumer(writeState, postings);

  synchronized(state) {
    state.put(id, postings);
  }
  return consumer;
}
 
Example 11
Source Project: lucene-solr   Source File: LuceneVarGapDocFreqInterval.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThreshold, termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example 12
Source Project: lucene-solr   Source File: LuceneFixedGap.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new FixedGapTermsIndexWriter(state, termIndexInterval);
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example 13
Source Project: lucene-solr   Source File: LuceneVarGapFixedInterval.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  PostingsWriterBase docs = new Lucene84PostingsWriter(state);

  // TODO: should we make the terms index more easily
  // pluggable?  Ie so that this codec would record which
  // index impl was used, and switch on loading?
  // Or... you must make a new Codec for this?
  TermsIndexWriterBase indexWriter;
  boolean success = false;
  try {
    indexWriter = new VariableGapTermsIndexWriter(state, new VariableGapTermsIndexWriter.EveryNTermSelector(termIndexInterval));
    success = true;
  } finally {
    if (!success) {
      docs.close();
    }
  }

  success = false;
  try {
    // Must use BlockTermsWriter (not BlockTree) because
    // BlockTree doens't support ords (yet)...
    FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs);
    success = true;
    return ret;
  } finally {
    if (!success) {
      try {
        docs.close();
      } finally {
        indexWriter.close();
      }
    }
  }
}
 
Example 14
Source Project: lucene-solr   Source File: CrankyPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from PostingsFormat.fieldsConsumer()");
  }  
  return new CrankyFieldsConsumer(delegate.fieldsConsumer(state), random);
}
 
Example 15
Source Project: lucene-solr   Source File: PerFieldPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(Fields fields, NormsProducer norms) throws IOException {
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(fields);

  // Write postings
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      // Exposes only the fields from this group:
      Fields maskedFields = new FilterFields(fields) {
        @Override
        public Iterator<String> iterator() {
          return group.fields.iterator();
        }
      };

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.write(maskedFields, norms);
    }
    success = true;
  } finally {
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 
Example 16
Source Project: lucene-solr   Source File: PerFieldPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void merge(MergeState mergeState, NormsProducer norms) throws IOException {
  @SuppressWarnings("unchecked") Iterable<String> indexedFieldNames = () ->
      new MergedIterator<>(true,
          Arrays.stream(mergeState.fieldsProducers).map(FieldsProducer::iterator).toArray(Iterator[]::new));
  Map<PostingsFormat, FieldsGroup> formatToGroups = buildFieldsGroupMapping(indexedFieldNames);

  // Merge postings
  PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
  boolean success = false;
  try {
    for (Map.Entry<PostingsFormat, FieldsGroup> ent : formatToGroups.entrySet()) {
      PostingsFormat format = ent.getKey();
      final FieldsGroup group = ent.getValue();

      FieldsConsumer consumer = format.fieldsConsumer(group.state);
      toClose.add(consumer);
      consumer.merge(pfMergeState.apply(group.fields), norms);
    }
    success = true;
  } finally {
    pfMergeState.reset();
    if (!success) {
      IOUtils.closeWhileHandlingException(toClose);
    }
  }
}
 
Example 17
Source Project: mtas   Source File: MtasCodecPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  if (delegatePostingsFormat != null) {
    return new MtasFieldsConsumer(
        delegatePostingsFormat.fieldsConsumer(state), state, getName(),
        delegatePostingsFormat.getName());
  } else {
    PostingsFormat pf = Codec.forName(delegateCodecName).postingsFormat();
    return pf.fieldsConsumer(state);
  }
}
 
Example 18
@Override
public FieldsConsumer consumer(final IndexOutput output) throws IOException {
    CodecUtil.writeHeader(output, CODEC_NAME, CODEC_VERSION_LATEST);
    return new FieldsConsumer() {
        private Map<String, Long> fieldOffsets = new HashMap<>();

        @Override
        public void close() throws IOException {
            try {
              /*
               * write the offsets per field such that we know where
               * we need to load the FSTs from
               */
                long pointer = output.getFilePointer();
                output.writeVInt(fieldOffsets.size());
                for (Map.Entry<String, Long> entry : fieldOffsets.entrySet()) {
                    output.writeString(entry.getKey());
                    output.writeVLong(entry.getValue());
                }
                output.writeLong(pointer);
                CodecUtil.writeFooter(output);
            } finally {
                IOUtils.close(output);
            }
        }

        @Override
        public void write(Fields fields) throws IOException {
            for(String field : fields) {
                Terms terms = fields.terms(field);
                if (terms == null) {
                    continue;
                }
                TermsEnum termsEnum = terms.iterator();
                PostingsEnum docsEnum = null;
                final SuggestPayload spare = new SuggestPayload();
                int maxAnalyzedPathsForOneInput = 0;
                final XAnalyzingSuggester.XBuilder builder = new XAnalyzingSuggester.XBuilder(maxSurfaceFormsPerAnalyzedForm, hasPayloads, XAnalyzingSuggester.PAYLOAD_SEP);
                int docCount = 0;
                while (true) {
                    BytesRef term = termsEnum.next();
                    if (term == null) {
                        break;
                    }
                    docsEnum = termsEnum.postings(docsEnum, PostingsEnum.PAYLOADS);
                    builder.startTerm(term);
                    int docFreq = 0;
                    while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
                        for (int i = 0; i < docsEnum.freq(); i++) {
                            final int position = docsEnum.nextPosition();
                            AnalyzingCompletionLookupProvider.this.parsePayload(docsEnum.getPayload(), spare);
                            builder.addSurface(spare.surfaceForm.get(), spare.payload.get(), spare.weight);
                            // multi fields have the same surface form so we sum up here
                            maxAnalyzedPathsForOneInput = Math.max(maxAnalyzedPathsForOneInput, position + 1);
                        }
                        docFreq++;
                        docCount = Math.max(docCount, docsEnum.docID()+1);
                    }
                    builder.finishTerm(docFreq);
                }
                /*
                 * Here we are done processing the field and we can
                 * buid the FST and write it to disk.
                 */
                FST<Pair<Long, BytesRef>> build = builder.build();
                assert build != null || docCount == 0: "the FST is null but docCount is != 0 actual value: [" + docCount + "]";
                /*
                 * it's possible that the FST is null if we have 2 segments that get merged
                 * and all docs that have a value in this field are deleted. This will cause
                 * a consumer to be created but it doesn't consume any values causing the FSTBuilder
                 * to return null.
                 */
                if (build != null) {
                    fieldOffsets.put(field, output.getFilePointer());
                    build.save(output);
                    /* write some more meta-info */
                    output.writeVInt(maxAnalyzedPathsForOneInput);
                    output.writeVInt(maxSurfaceFormsPerAnalyzedForm);
                    output.writeInt(maxGraphExpansions); // can be negative
                    int options = 0;
                    options |= preserveSep ? SERIALIZE_PRESERVE_SEPARATORS : 0;
                    options |= hasPayloads ? SERIALIZE_HAS_PAYLOADS : 0;
                    options |= preservePositionIncrements ? SERIALIZE_PRESERVE_POSITION_INCREMENTS : 0;
                    output.writeVInt(options);
                    output.writeVInt(XAnalyzingSuggester.SEP_LABEL);
                    output.writeVInt(XAnalyzingSuggester.END_BYTE);
                    output.writeVInt(XAnalyzingSuggester.PAYLOAD_SEP);
                    output.writeVInt(XAnalyzingSuggester.HOLE_CHARACTER);
                }
            }
        }
    };
}
 
Example 19
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
    throw new UnsupportedOperationException("this codec can only be used for reading");
}
 
Example 20
Source Project: lucene-solr   Source File: DirectPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return PostingsFormat.forName("Lucene84").fieldsConsumer(state);
}
 
Example 21
Source Project: lucene-solr   Source File: SimpleTextPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new SimpleTextFieldsWriter(state);
}
 
Example 22
Source Project: lucene-solr   Source File: BloomFilteringPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
public BloomFilteredFieldsConsumer(FieldsConsumer fieldsConsumer,
    SegmentWriteState state) {
  this.delegateFieldsConsumer = fieldsConsumer;
  this.state = state;
}
 
Example 23
Source Project: lucene-solr   Source File: UniformSplitPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new UniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 
Example 24
Source Project: lucene-solr   Source File: STUniformSplitPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected FieldsConsumer createUniformSplitTermsWriter(PostingsWriterBase postingsWriter, SegmentWriteState state,
                                             int targetNumBlockLines, int deltaNumLines, BlockEncoder blockEncoder) throws IOException {
  return new STUniformSplitTermsWriter(postingsWriter, state, targetNumBlockLines, deltaNumLines, blockEncoder);
}
 
Example 25
Source Project: lucene-solr   Source File: AssertingPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException {
  return new AssertingFieldsConsumer(state, in.fieldsConsumer(state));
}
 
Example 26
Source Project: lucene-solr   Source File: AssertingPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
AssertingFieldsConsumer(SegmentWriteState writeState, FieldsConsumer in) {
  this.writeState = writeState;
  this.in = in;
}
 
Example 27
@Override
public FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return delegate.fieldsConsumer(state);
}
 
Example 28
Source Project: lucene-solr   Source File: CrankyPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
CrankyFieldsConsumer(FieldsConsumer delegate, Random random) {
  this.delegate = delegate;
  this.random = random;
}
 
Example 29
Source Project: lucene-solr   Source File: PerFieldPostingsFormat.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final FieldsConsumer fieldsConsumer(SegmentWriteState state)
    throws IOException {
  return new FieldsWriter(state);
}
 
Example 30
Source Project: lucene-solr   Source File: SegmentMerger.java    License: Apache License 2.0 4 votes vote down vote up
private void mergeTerms(SegmentWriteState segmentWriteState, NormsProducer norms) throws IOException {
  try (FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState)) {
    consumer.merge(mergeState, norms);
  }
}