Java Code Examples for org.apache.lucene.index.FieldInfo

The following examples show how to use org.apache.lucene.index.FieldInfo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: crate   Source File: RecoverySourcePruneMergePolicy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StoredFieldsReader getFieldsReader() {
    StoredFieldsReader fieldsReader = super.getFieldsReader();
    return new FilterStoredFieldsReader(fieldsReader) {
        @Override
        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
            if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
                super.visitDocument(docID, visitor);
            } else {
                super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
                    @Override
                    public Status needsField(FieldInfo fieldInfo) throws IOException {
                        if (recoverySourceField.equals(fieldInfo.name)) {
                            return Status.NO;
                        }
                        return super.needsField(fieldInfo);
                    }
                });
            }
        }
    };
}
 
Example 2
Source Project: lucene-solr   Source File: PointsWriter.java    License: Apache License 2.0 6 votes vote down vote up
/** Default merge implementation to merge incoming points readers by visiting all their points and
 *  adding to this writer */
public void merge(MergeState mergeState) throws IOException {
  // check each incoming reader
  for (PointsReader reader : mergeState.pointsReaders) {
    if (reader != null) {
      reader.checkIntegrity();
    }
  }
  // merge field at a time
  for (FieldInfo fieldInfo : mergeState.mergeFieldInfos) {
    if (fieldInfo.getPointDimensionCount() != 0) {
      mergeOneField(mergeState, fieldInfo);
    }
  }
  finish();
}
 
Example 3
Source Project: lucene-solr   Source File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
  if (valuesIn == null) {
    valuesIn = reader.getSortedDocValues(field);
  }

  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  }

  final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info == null) {
    return DocValues.emptyBinary();
  } else if (info.getDocValuesType() != DocValuesType.NONE) {
    throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
  } else if (info.getIndexOptions() == IndexOptions.NONE) {
    return DocValues.emptyBinary();
  }

  BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
  return impl.iterator();
}
 
Example 4
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
  SortedSetDocValues sortedSetDocValues = _sortedSetDocValuesCache.get(field.number);
  if (sortedSetDocValues != null) {
    return sortedSetDocValues;
  }
  synchronized (_sortedSetDocValuesCache) {
    sortedSetDocValues = _sortedSetDocValuesCache.get(field.number);
    if (sortedSetDocValues != null) {
      return sortedSetDocValues;
    }
    sortedSetDocValues = newSortedSetDocValues(field);
    if (_cache && sortedSetDocValues != null) {
      _sortedSetDocValuesCache.put(field.number, sortedSetDocValues);
    }
    return sortedSetDocValues;
  }
}
 
Example 5
Source Project: lucene-solr   Source File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 
Example 6
@Override
public void addSortedSetField(FieldInfo field, Iterable<BytesRef> values, Iterable<Number> docToOrdCount, Iterable<Number> ords) throws IOException {
  meta.writeVInt(field.number);
  meta.writeByte(DiskDocValuesFormat.SORTED_SET);
  // write the ord -> byte[] as a binary field
  addBinaryField(field, values);
  // write the stream of ords as a numeric field
  // NOTE: we could return an iterator that delta-encodes these within a doc
  addNumericField(field, ords);
  
  // write the doc -> ord count as a absolute index to the stream
  meta.writeVInt(field.number);
  meta.writeByte(DiskDocValuesFormat.NUMERIC);
  meta.writeVInt(PackedInts.VERSION_CURRENT);
  meta.writeLong(data.getFilePointer());
  meta.writeVLong(maxDoc);
  meta.writeVInt(BLOCK_SIZE);

  final MonotonicBlockPackedWriter writer = new MonotonicBlockPackedWriter(data, BLOCK_SIZE);
  long addr = 0;
  for (Number v : docToOrdCount) {
    addr += v.longValue();
    writer.add(addr);
  }
  writer.finish();
}
 
Example 7
Source Project: lucene-solr   Source File: FacetFieldProcessorByHashDV.java    License: Apache License 2.0 6 votes vote down vote up
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) {
  super(fcontext, freq, sf);
  if (freq.mincount == 0) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support mincount=0");
  }
  if (freq.prefix != null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support prefix"); // yet, but it could
  }
  FieldInfo fieldInfo = fcontext.searcher.getFieldInfos().fieldInfo(sf.getName());
  if (fieldInfo != null &&
      fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED_NUMERIC) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" only support single valued number/string with docValues");
  }
}
 
Example 8
Source Project: lucene-solr   Source File: Lucene84PostingsReader.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public ImpactsEnum impacts(FieldInfo fieldInfo, BlockTermState state, int flags) throws IOException {
  if (state.docFreq <= BLOCK_SIZE) {
    // no skip data
    return new SlowImpactsEnum(postings(fieldInfo, state, null, flags));
  }

  final boolean indexHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  final boolean indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  final boolean indexHasPayloads = fieldInfo.hasPayloads();

  if (indexHasPositions == false || PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) == false) {
    return new BlockImpactsDocsEnum(fieldInfo, (IntBlockTermState) state);
  }

  if (indexHasPositions &&
      PostingsEnum.featureRequested(flags, PostingsEnum.POSITIONS) &&
      (indexHasOffsets == false || PostingsEnum.featureRequested(flags, PostingsEnum.OFFSETS) == false) &&
      (indexHasPayloads == false || PostingsEnum.featureRequested(flags, PostingsEnum.PAYLOADS) == false)) {
    return new BlockImpactsPostingsEnum(fieldInfo, (IntBlockTermState) state);
  }

  return new BlockImpactsEverythingEnum(fieldInfo, (IntBlockTermState) state, flags);
}
 
Example 9
public void stringFieldRead(final FieldInfo fieldInfo, final byte[] fieldValue) {
    try {
        if(!recordField(fieldInfo.name, true)) {
            return;
        }
        fieldRead0(fieldInfo.name, new String(fieldValue, StandardCharsets.UTF_8));
    } catch (Exception e) {
        log.error("Unexpected error reading string field '{}' in index '{}'", fieldInfo.name, index.getName());
    }
}
 
Example 10
Source Project: lucene-solr   Source File: Lucene84PostingsReader.java    License: Apache License 2.0 5 votes vote down vote up
public BlockDocsEnum(FieldInfo fieldInfo) throws IOException {
  this.startDocIn = Lucene84PostingsReader.this.docIn;
  this.docIn = null;
  indexHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
  indexHasPos = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
  indexHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
  indexHasPayloads = fieldInfo.hasPayloads();
  // We set the last element of docBuffer to NO_MORE_DOCS, it helps save conditionals in advance()
  docBuffer[BLOCK_SIZE] = NO_MORE_DOCS;
}
 
Example 11
Source Project: lucene-solr   Source File: DocValuesConsumer.java    License: Apache License 2.0 5 votes vote down vote up
/** Merges in the fields from the readers in 
 *  <code>mergeState</code>. The default implementation 
 *  calls {@link #mergeNumericField}, {@link #mergeBinaryField},
 *  {@link #mergeSortedField}, {@link #mergeSortedSetField},
 *  or {@link #mergeSortedNumericField} for each field,
 *  depending on its type.
 *  Implementations can override this method 
 *  for more sophisticated merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
  for(DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {
    if (docValuesProducer != null) {
      docValuesProducer.checkIntegrity();
    }
  }

  for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
    DocValuesType type = mergeFieldInfo.getDocValuesType();
    if (type != DocValuesType.NONE) {
      if (type == DocValuesType.NUMERIC) {
        mergeNumericField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.BINARY) {
        mergeBinaryField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED) {
        mergeSortedField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_SET) {
        mergeSortedSetField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_NUMERIC) {
        mergeSortedNumericField(mergeFieldInfo, mergeState);
      } else {
        throw new AssertionError("type=" + type);
      }
    }
  }
}
 
Example 12
Source Project: lucene-solr   Source File: Lucene80NormsProducer.java    License: Apache License 2.0 5 votes vote down vote up
private RandomAccessInput getDisiJumpTable(FieldInfo field, NormsEntry entry) throws IOException {
   RandomAccessInput jumpTable = null;
   if (merging) {
     jumpTable = disiJumpTables.get(field.number);
   }
   if (jumpTable == null) {
     jumpTable = IndexedDISI.createJumpTable(
         data, entry.docsWithFieldOffset, entry.docsWithFieldLength, entry.jumpTableEntryCount);
     if (merging) {
       disiJumpTables.put(field.number, jumpTable);
     }
   }
   return jumpTable;
}
 
Example 13
Source Project: lucene-solr   Source File: Lucene60FieldInfosFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
      output.writeVInt(fi.getPointDimensionCount());
      if (fi.getPointDimensionCount() != 0) {
        output.writeVInt(fi.getPointIndexDimensionCount());
        output.writeVInt(fi.getPointNumBytes());
      }
    }
    CodecUtil.writeFooter(output);
  }
}
 
Example 14
Source Project: lucene-solr   Source File: SimpleTextPointsReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public PointValues getValues(String fieldName) throws IOException {
  FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
  if (fieldInfo == null) {
    throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
  }
  if (fieldInfo.getPointDimensionCount() == 0) {
    throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index points");
  }
  return readers.get(fieldName);
}
 
Example 15
Source Project: lucene-solr   Source File: RangeFieldQuery.java    License: Apache License 2.0 5 votes vote down vote up
/** Check indexed field info against the provided query data. */
private void checkFieldInfo(FieldInfo fieldInfo) {
  if (fieldInfo.getPointDimensionCount()/2 != numDims) {
    throw new IllegalArgumentException("field=\"" + field + "\" was indexed with numDims="
        + fieldInfo.getPointDimensionCount()/2 + " but this query has numDims=" + numDims);
  }
}
 
Example 16
Source Project: lucene-solr   Source File: IndexSizeEstimator.java    License: Apache License 2.0 5 votes vote down vote up
/** Process a string field. */
public void stringField(FieldInfo fieldInfo, String value) throws IOException {
  // trim the value if needed
  int len = value != null ? UnicodeUtil.calcUTF16toUTF8Length(value, 0, value.length()) : 0;
  if (value.length() > maxLength) {
    value = value.substring(0, maxLength);
  }
  countItem(fieldInfo.name, value, len);
}
 
Example 17
Source Project: lucene-solr   Source File: SolrDocumentFetcher.java    License: Apache License 2.0 5 votes vote down vote up
private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) {
  if (!schemaField.hasDocValues() || !schemaField.stored()) return false;
  if (schemaField.multiValued()) return false;
  DocValuesType docValuesType = fieldInfo.getDocValuesType();
  NumberType numberType = schemaField.getType().getNumberType();
  // can not decode a numeric without knowing its numberType
  if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) {
    return false;
  }
  return true;
}
 
Example 18
Source Project: lucene-solr   Source File: CrankyDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addNumericField()");
  }
  delegate.addNumericField(field, valuesProducer);
}
 
Example 19
Source Project: lucene-solr   Source File: AssertingDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
  if (merging) {
    AssertingCodec.assertThread("DocValuesProducer", creationThread);
  }
  assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
  SortedNumericDocValues values = in.getSortedNumeric(field);
  assert values != null;
  return new AssertingLeafReader.AssertingSortedNumericDocValues(values, maxDoc);
}
 
Example 20
Source Project: lucene-solr   Source File: DeltaBaseTermStateSerializer.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reads a {@link BlockTermState} from the provided {@link DataInput}.
 * <p>
 * Simpler variant of {@link Lucene84PostingsReader#decodeTerm(DataInput, FieldInfo, BlockTermState, boolean)}.
 *
 * @param reuse {@link BlockTermState} to reuse; or null to create a new one.
 */
public BlockTermState readTermState(long baseDocStartFP, long basePosStartFP, long basePayStartFP,
                                    DataInput termStatesInput, FieldInfo fieldInfo, BlockTermState reuse) throws IOException {
  IndexOptions indexOptions = fieldInfo.getIndexOptions();
  boolean hasFreqs = indexOptions != IndexOptions.DOCS;
  boolean hasPositions = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;

  IntBlockTermState intTermState = reuse != null ? reset((IntBlockTermState) reuse) : new IntBlockTermState();

  intTermState.docFreq = termStatesInput.readVInt();
  intTermState.totalTermFreq = hasFreqs ?
      intTermState.docFreq + termStatesInput.readVLong() : intTermState.docFreq;
  assert intTermState.totalTermFreq >= intTermState.docFreq;

  if (intTermState.docFreq == 1) {
    intTermState.singletonDocID = termStatesInput.readVInt();
  } else {
    intTermState.docStartFP = baseDocStartFP + termStatesInput.readVLong();
  }

  if (hasPositions) {
    intTermState.posStartFP = basePosStartFP + termStatesInput.readVLong();
    boolean hasOffsets = indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
    if (hasOffsets || fieldInfo.hasPayloads()) {
      intTermState.payStartFP = basePayStartFP + termStatesInput.readVLong();
    }
    if (intTermState.totalTermFreq > BLOCK_SIZE) {
      intTermState.lastPosBlockOffset = termStatesInput.readVLong();
    }
  }
  if (intTermState.docFreq > BLOCK_SIZE) {
    intTermState.skipOffset = termStatesInput.readVLong();
  }
  return intTermState;
}
 
Example 21
Source Project: lucene-solr   Source File: CompletionFieldsConsumer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void close() throws IOException {
  if (closed) {
    return;
  }
  closed = true;
  String indexFile = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, INDEX_EXTENSION);
  boolean success = false;
  try (IndexOutput indexOut = state.directory.createOutput(indexFile, state.context)) {
    delegateFieldsConsumer.close();
    CodecUtil.writeIndexHeader(indexOut, codecName, COMPLETION_VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
    /*
     * we write the delegate postings format name so we can load it
     * without getting an instance in the ctor
     */
    indexOut.writeString(delegatePostingsFormatName);
    // write # of seen fields
    indexOut.writeVInt(seenFields.size());
    // write field numbers and dictOut offsets
    for (Map.Entry<String, CompletionMetaData> seenField : seenFields.entrySet()) {
      FieldInfo fieldInfo = state.fieldInfos.fieldInfo(seenField.getKey());
      indexOut.writeVInt(fieldInfo.number);
      CompletionMetaData metaData = seenField.getValue();
      indexOut.writeVLong(metaData.filePointer);
      indexOut.writeVLong(metaData.minWeight);
      indexOut.writeVLong(metaData.maxWeight);
      indexOut.writeByte(metaData.type);
    }
    CodecUtil.writeFooter(indexOut);
    CodecUtil.writeFooter(dictOut);
    IOUtils.close(dictOut);
    success = true;
  } finally {
    if (success == false) {
      IOUtils.closeWhileHandlingException(dictOut, delegateFieldsConsumer);
    }
  }
}
 
Example 22
Source Project: lucene-solr   Source File: CrankyDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedSetField()");
  }
  delegate.addSortedSetField(field, valuesProducer);
}
 
Example 23
Source Project: lucene-solr   Source File: IDVersionPostingsWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setField(FieldInfo fieldInfo) {
  super.setField(fieldInfo);
  if (fieldInfo.getIndexOptions() != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
    throw new IllegalArgumentException("field must be index using IndexOptions.DOCS_AND_FREQS_AND_POSITIONS");
  }
  // LUCENE-5693: because CheckIndex cross-checks term vectors with postings even for deleted docs, and because our PF only indexes the
  // non-deleted documents on flush, CheckIndex will see this as corruption:
  if (fieldInfo.hasVectors()) {
    throw new IllegalArgumentException("field cannot index term vectors: CheckIndex will report this as index corruption");
  }
  lastState = emptyState;
}
 
Example 24
Source Project: lucene-solr   Source File: BloomFilteringPostingsFormat.java    License: Apache License 2.0 5 votes vote down vote up
private void saveAppropriatelySizedBloomFilter(IndexOutput bloomOutput,
    FuzzySet bloomFilter, FieldInfo fieldInfo) throws IOException {
  
  FuzzySet rightSizedSet = bloomFilterFactory.downsize(fieldInfo,
      bloomFilter);
  if (rightSizedSet == null) {
    rightSizedSet = bloomFilter;
  }
  rightSizedSet.serialize(bloomOutput);
}
 
Example 25
Source Project: lucene-solr   Source File: FieldMetadata.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs field metadata for reading or writing.
 * @param maxDoc The total number of documents in the segment being written.
 * @param isMutable Set true if this FieldMetadata is created for writing the index. Set false if it is used for reading the index.
 */
protected FieldMetadata(FieldInfo fieldInfo, int maxDoc, boolean isMutable) {
  assert isMutable || maxDoc == 0;
  this.fieldInfo = fieldInfo;
  this.isMutable = isMutable;
  // docsSeen must not be set if this FieldMetadata is immutable, that means it is used for reading the index.
  this.docsSeen = isMutable ? new FixedBitSet(maxDoc) : null;
  this.dictionaryStartFP = -1;
  this.firstBlockStartFP = -1;
  this.lastBlockStartFP = -1;
}
 
Example 26
Source Project: lucene-solr   Source File: Lucene84PostingsWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void encodeTerm(DataOutput out, FieldInfo fieldInfo, BlockTermState _state, boolean absolute) throws IOException {
  IntBlockTermState state = (IntBlockTermState)_state;
  if (absolute) {
    lastState = emptyState;
    assert lastState.docStartFP == 0;
  }

  if (lastState.singletonDocID != -1 && state.singletonDocID != -1 && state.docStartFP == lastState.docStartFP) {
    // With runs of rare values such as ID fields, the increment of pointers in the docs file is often 0.
    // Furthermore some ID schemes like auto-increment IDs or Flake IDs are monotonic, so we encode the delta
    // between consecutive doc IDs to save space.
    final long delta = (long) state.singletonDocID - lastState.singletonDocID;
    out.writeVLong((BitUtil.zigZagEncode(delta) << 1) | 0x01);
  } else {
    out.writeVLong((state.docStartFP - lastState.docStartFP) << 1);
    if (state.singletonDocID != -1) {
      out.writeVInt(state.singletonDocID);
    }
  }

  if (writePositions) {
    out.writeVLong(state.posStartFP - lastState.posStartFP);
    if (writePayloads || writeOffsets) {
      out.writeVLong(state.payStartFP - lastState.payStartFP);
    }
  }
  if (writePositions) {
    if (state.lastPosBlockOffset != -1) {
      out.writeVLong(state.lastPosBlockOffset);
    }
  }
  if (state.skipOffset != -1) {
    out.writeVLong(state.skipOffset);
  }
  lastState = state;
}
 
Example 27
Source Project: lucene-solr   Source File: Lucene50FieldInfosFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
  final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
  try (IndexOutput output = directory.createOutput(fileName, context)) {
    CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
    output.writeVInt(infos.size());
    for (FieldInfo fi : infos) {
      fi.checkConsistency();

      output.writeString(fi.name);
      output.writeVInt(fi.number);

      byte bits = 0x0;
      if (fi.hasVectors()) bits |= STORE_TERMVECTOR;
      if (fi.omitsNorms()) bits |= OMIT_NORMS;
      if (fi.hasPayloads()) bits |= STORE_PAYLOADS;
      output.writeByte(bits);

      output.writeByte(indexOptionsByte(fi.getIndexOptions()));

      // pack the DV type and hasNorms in one byte
      output.writeByte(docValuesByte(fi.getDocValuesType()));
      output.writeLong(fi.getDocValuesGen());
      output.writeMapOfStrings(fi.attributes());
    }
    CodecUtil.writeFooter(output);
  }
}
 
Example 28
Source Project: incubator-retired-blur   Source File: IndexManager.java    License: Apache License 2.0 5 votes vote down vote up
private static String getRowId(IndexReader reader, int docId) throws CorruptIndexException, IOException {
  reader.document(docId, new StoredFieldVisitor() {
    @Override
    public Status needsField(FieldInfo fieldInfo) throws IOException {
      if (ROW_ID.equals(fieldInfo.name)) {
        return StoredFieldVisitor.Status.STOP;
      }
      return StoredFieldVisitor.Status.NO;
    }
  });
  return reader.document(docId).get(ROW_ID);
}
 
Example 29
Source Project: lucene-solr   Source File: Lucene60PointsReader.java    License: Apache License 2.0 5 votes vote down vote up
/** Returns the underlying {@link BKDReader}.
 *
 * @lucene.internal */
@Override
public PointValues getValues(String fieldName) {
  FieldInfo fieldInfo = readState.fieldInfos.fieldInfo(fieldName);
  if (fieldInfo == null) {
    throw new IllegalArgumentException("field=\"" + fieldName + "\" is unrecognized");
  }
  if (fieldInfo.getPointDimensionCount() == 0) {
    throw new IllegalArgumentException("field=\"" + fieldName + "\" did not index point values");
  }

  return readers.get(fieldInfo.number);
}
 
Example 30
Source Project: lucene-solr   Source File: StoredFieldsWriter.java    License: Apache License 2.0 5 votes vote down vote up
void reset(FieldInfo field) {
  if (remapper != null) {
    // field numbers are not aligned, we need to remap to the new field number
    currentField = remapper.fieldInfo(field.name);
  } else {
    currentField = field;
  }
  binaryValue = null;
  stringValue = null;
  numericValue = null;
}