Java Code Examples for org.apache.lucene.index.FieldInfo#getDocValuesType()

The following examples show how to use org.apache.lucene.index.FieldInfo#getDocValuesType() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FacetFieldProcessorByHashDV.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) {
  super(fcontext, freq, sf);
  if (freq.mincount == 0) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support mincount=0");
  }
  if (freq.prefix != null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" doesn't support prefix"); // yet, but it could
  }
  FieldInfo fieldInfo = fcontext.searcher.getFieldInfos().fieldInfo(sf.getName());
  if (fieldInfo != null &&
      fieldInfo.getDocValuesType() != DocValuesType.NUMERIC &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED &&
      fieldInfo.getDocValuesType() != DocValuesType.SORTED_NUMERIC) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        getClass()+" only support single valued number/string with docValues");
  }
}
 
Example 2
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  BinaryDocValues valuesIn = reader.getBinaryDocValues(field);
  if (valuesIn == null) {
    valuesIn = reader.getSortedDocValues(field);
  }

  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  }

  final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info == null) {
    return DocValues.emptyBinary();
  } else if (info.getDocValuesType() != DocValuesType.NONE) {
    throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
  } else if (info.getIndexOptions() == IndexOptions.NONE) {
    return DocValues.emptyBinary();
  }

  BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
  return impl.iterator();
}
 
Example 3
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 
Example 4
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException {
  final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  if (fieldInfo == null) {
    // field does not exist or has no value
    return new Bits.MatchNoBits(reader.maxDoc());
  } 
  
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
    // doc values case
  } else if (parser instanceof PointParser) {
    // points case
  } else {
    // postings case
    if (fieldInfo.getIndexOptions() == IndexOptions.NONE) {
      return new Bits.MatchNoBits(reader.maxDoc());
    }
  }
  BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser));
  return bitsEntry.bits;
}
 
Example 5
Source File: VersionFieldUpgrader.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
static CodecReader wrap(CodecReader reader) throws IOException {
    final FieldInfos fieldInfos = reader.getFieldInfos();
    final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME);
    if (versionInfo != null && versionInfo.getDocValuesType() != DocValuesType.NONE) {
        // the reader is a recent one, it has versions and they are stored
        // in a numeric doc values field
        return reader;
    }
    // The segment is an old one, look at the _uid field
    final Terms terms = reader.terms(UidFieldMapper.NAME);
    if (terms == null || !terms.hasPayloads()) {
        // The segment doesn't have an _uid field or doesn't have payloads
        // don't try to do anything clever. If any other segment has versions
        // all versions of this segment will be initialized to 0
        return reader;
    }
    // convert _uid payloads -> _version docvalues
    return new VersionFieldUpgrader(reader);
}
 
Example 6
Source File: LatLonDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */
static void checkCompatible(FieldInfo fieldInfo) {
  // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) {
    throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + 
                                       " but this type has docValuesType=" + TYPE.docValuesType() + 
                                       ", is the field really a LatLonDocValuesField?");
  }
}
 
Example 7
Source File: SolrDocumentFetcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) {
  if (!schemaField.hasDocValues() || !schemaField.stored()) return false;
  if (schemaField.multiValued()) return false;
  DocValuesType docValuesType = fieldInfo.getDocValuesType();
  NumberType numberType = schemaField.getType().getNumberType();
  // can not decode a numeric without knowing its numberType
  if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) {
    return false;
  }
  return true;
}
 
Example 8
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private BitsEntry createValueDocValues(LeafReader reader, String field) throws IOException {
  FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  
  DocValuesType dvType = fieldInfo.getDocValuesType();
  DocIdSetIterator iterator;
  switch(dvType) {
  case NUMERIC:
    iterator = reader.getNumericDocValues(field);
    break;
  case BINARY:
    iterator = reader.getBinaryDocValues(field);
    break;
  case SORTED:
    iterator = reader.getSortedDocValues(field);
    break;
  case SORTED_NUMERIC:
    iterator = reader.getSortedNumericDocValues(field);
    break;
  case SORTED_SET:
    iterator = reader.getSortedSetDocValues(field);
    break;
  default:
    throw new AssertionError();
  }

  FixedBitSet bits = new FixedBitSet(reader.maxDoc());
  while (true) {
    int docID = iterator.nextDoc();
    if (docID == DocIdSetIterator.NO_MORE_DOCS) {
      break;
    }
    bits.set(docID);
  }

  return new BitsEntry(bits);
}
 
Example 9
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected BitsEntry createValue(LeafReader reader, CacheKey key) throws IOException {
  final String field = key.field;
  final Parser parser = (Parser) key.custom;
  FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
    return createValueDocValues(reader, field);
  } else if (parser instanceof PointParser) {
    return createValuePoints(reader, field);
  } else {
    return createValuePostings(reader, field);
  }
}
 
Example 10
Source File: XYDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** helper: checks a fieldinfo and throws exception if its definitely not a XYDocValuesField */
static void checkCompatible(FieldInfo fieldInfo) {
  // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) {
    throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + 
                                       " but this type has docValuesType=" + TYPE.docValuesType() + 
                                       ", is the field really a XYDocValuesField?");
  }
}
 
Example 11
Source File: UninvertDocValuesMergePolicyFactory.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private UninvertingReader.Type getUninversionType(FieldInfo fi) {
  SchemaField sf = schema.getFieldOrNull(fi.name);
  
  if (null != sf &&
      sf.hasDocValues() &&
      fi.getDocValuesType() == DocValuesType.NONE &&
      fi.getIndexOptions() != IndexOptions.NONE) {
    return sf.getType().getUninversionType(sf);
  } else {
    return null;
  }
}
 
Example 12
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public FieldsReader(final SegmentReadState readState) throws IOException {

      // Init each unique format:
      boolean success = false;
      try {
        // Read field name -> format name
        for (FieldInfo fi : readState.fieldInfos) {
          if (fi.getDocValuesType() != DocValuesType.NONE) {
            final String fieldName = fi.name;
            final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY);
            if (formatName != null) {
              // null formatName means the field is in fieldInfos, but has no docvalues!
              final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY);
              if (suffix == null) {
                throw new IllegalStateException("missing attribute: " + PER_FIELD_SUFFIX_KEY + " for field: " + fieldName);
              }
              DocValuesFormat format = DocValuesFormat.forName(formatName);
              String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix));
              if (!formats.containsKey(segmentSuffix)) {
                formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
              }
              fields.put(fieldName, formats.get(segmentSuffix));
            }
          }
        }
        success = true;
      } finally {
        if (!success) {
          IOUtils.closeWhileHandlingException(formats.values());
        }
      }
    }
 
Example 13
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void merge(MergeState mergeState) throws IOException {
  Map<DocValuesConsumer, Collection<String>> consumersToField = new IdentityHashMap<>();

  // Group each consumer by the fields it handles
  for (FieldInfo fi : mergeState.mergeFieldInfos) {
    if (fi.getDocValuesType() == DocValuesType.NONE) {
      continue;
    }
    // merge should ignore current format for the fields being merged
    DocValuesConsumer consumer = getInstance(fi, true);
    Collection<String> fieldsForConsumer = consumersToField.get(consumer);
    if (fieldsForConsumer == null) {
      fieldsForConsumer = new ArrayList<>();
      consumersToField.put(consumer, fieldsForConsumer);
    }
    fieldsForConsumer.add(fi.name);
  }

  // Delegate the merge to the appropriate consumer
  PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState);
  try {
    for (Map.Entry<DocValuesConsumer, Collection<String>> e : consumersToField.entrySet()) {
      e.getKey().merge(pfMergeState.apply(e.getValue()));
    }
  } finally {
    pfMergeState.reset();
  }
}
 
Example 14
Source File: PerFieldMergeState.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
FilterFieldInfos(FieldInfos src, Collection<String> filterFields) {
  // Copy all the input FieldInfo objects since the field numbering must be kept consistent
  super(toArray(src));

  boolean hasVectors = false;
  boolean hasProx = false;
  boolean hasPayloads = false;
  boolean hasOffsets = false;
  boolean hasFreq = false;
  boolean hasNorms = false;
  boolean hasDocValues = false;
  boolean hasPointValues = false;

  this.filteredNames = new HashSet<>(filterFields);
  this.filtered = new ArrayList<>(filterFields.size());
  for (FieldInfo fi : src) {
    if (this.filteredNames.contains(fi.name)) {
      this.filtered.add(fi);
      hasVectors |= fi.hasVectors();
      hasProx |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0;
      hasFreq |= fi.getIndexOptions() != IndexOptions.DOCS;
      hasOffsets |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
      hasNorms |= fi.hasNorms();
      hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE;
      hasPayloads |= fi.hasPayloads();
      hasPointValues |= (fi.getPointDimensionCount() != 0);
    }
  }

  this.filteredHasVectors = hasVectors;
  this.filteredHasProx = hasProx;
  this.filteredHasPayloads = hasPayloads;
  this.filteredHasOffsets = hasOffsets;
  this.filteredHasFreq = hasFreq;
  this.filteredHasNorms = hasNorms;
  this.filteredHasDocValues = hasDocValues;
  this.filteredHasPointValues = hasPointValues;
}
 
Example 15
Source File: DocValuesConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Merges in the fields from the readers in 
 *  <code>mergeState</code>. The default implementation 
 *  calls {@link #mergeNumericField}, {@link #mergeBinaryField},
 *  {@link #mergeSortedField}, {@link #mergeSortedSetField},
 *  or {@link #mergeSortedNumericField} for each field,
 *  depending on its type.
 *  Implementations can override this method 
 *  for more sophisticated merging (bulk-byte copying, etc). */
public void merge(MergeState mergeState) throws IOException {
  for(DocValuesProducer docValuesProducer : mergeState.docValuesProducers) {
    if (docValuesProducer != null) {
      docValuesProducer.checkIntegrity();
    }
  }

  for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) {
    DocValuesType type = mergeFieldInfo.getDocValuesType();
    if (type != DocValuesType.NONE) {
      if (type == DocValuesType.NUMERIC) {
        mergeNumericField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.BINARY) {
        mergeBinaryField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED) {
        mergeSortedField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_SET) {
        mergeSortedSetField(mergeFieldInfo, mergeState);
      } else if (type == DocValuesType.SORTED_NUMERIC) {
        mergeSortedNumericField(mergeFieldInfo, mergeState);
      } else {
        throw new AssertionError("type=" + type);
      }
    }
  }
}
 
Example 16
Source File: DocumentField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static DocumentField of(FieldInfo finfo, IndexableField field, IndexReader reader, int docId)
    throws IOException {

  Objects.requireNonNull(finfo);
  Objects.requireNonNull(reader);

  DocumentField dfield = new DocumentField();

  dfield.name = finfo.name;
  dfield.idxOptions = finfo.getIndexOptions();
  dfield.hasTermVectors = finfo.hasVectors();
  dfield.hasPayloads = finfo.hasPayloads();
  dfield.hasNorms = finfo.hasNorms();

  if (finfo.hasNorms()) {
    NumericDocValues norms = MultiDocValues.getNormValues(reader, finfo.name);
    if (norms.advanceExact(docId)) {
      dfield.norm = norms.longValue();
    }
  }

  dfield.dvType = finfo.getDocValuesType();

  dfield.pointDimensionCount = finfo.getPointDimensionCount();
  dfield.pointNumBytes = finfo.getPointNumBytes();

  if (field != null) {
    dfield.isStored = field.fieldType().stored();
    dfield.stringValue = field.stringValue();
    if (field.binaryValue() != null) {
      dfield.binaryValue = BytesRef.deepCopyOf(field.binaryValue());
    }
    dfield.numericValue = field.numericValue();
  }

  return dfield;
}
 
Example 17
Source File: Geo3DDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** helper: checks a fieldinfo and throws exception if its definitely not a Geo3DDocValuesField */
static void checkCompatible(FieldInfo fieldInfo) {
  // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment.
  if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) {
    throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + 
                                       " but this type has docValuesType=" + TYPE.docValuesType() + 
                                       ", is the field really a Geo3DDocValuesField?");
  }
}
 
Example 18
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException {
  if (parser == null) {
    throw new NullPointerException();
  }
  final NumericDocValues valuesIn = reader.getNumericDocValues(field);
  if (valuesIn != null) {
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptyNumeric();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    }
    
    if (parser instanceof PointParser) {
      // points case
      // no points in this segment
      if (info.getPointDimensionCount() == 0) {
        return DocValues.emptyNumeric();
      }
      if (info.getPointDimensionCount() != 1) {
        throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount());
      }
      PointValues values = reader.getPointValues(field);
      // no actual points for this field (e.g. all points deleted)
      if (values == null || values.size() == 0) {
        return DocValues.emptyNumeric();
      }
      // not single-valued
      if (values.size() != values.getDocCount()) {
        throw new IllegalStateException("Type mismatch: " + field + " was indexed with multiple values, numValues=" + values.size() + ",numDocs=" + values.getDocCount());
      }
    } else {
      // postings case 
      // not indexed
      if (info.getIndexOptions() == IndexOptions.NONE) {
        return DocValues.emptyNumeric();
      }
    }

    return ((LongsFromArray) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser))).iterator();
  }
}
 
Example 19
Source File: TestRetrieveFieldsOptimizer.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
void expectedValsAsStrings(final FieldInfo info, List<String> valsAsStrings) {
  if (schemaField.stored() || schemaField.multiValued() == false) {
    return ;
  }

  switch (info.getDocValuesType()) {
    case NONE: // These three types are single values, just return.
    case NUMERIC:
    case BINARY: // here for completeness, really doesn't make sense.
      return;

    case SORTED_NUMERIC: // Can have multiple, identical values. This was a surprise to me.
      break;

    case SORTED_SET: // Obey set semantics.
    case SORTED:
      Set<String> uniq = new TreeSet<>(valsAsStrings);
      valsAsStrings.clear();
      valsAsStrings.addAll(uniq);
      break;
  }

  // Now order them if string-based comparison isn't reasonable
  switch (testFieldType.getSolrTypeClass()) {
    case "solr.TrieIntField":
    case "solr.TrieLongField":

      Collections.sort(valsAsStrings, Comparator.comparingInt(Integer::parseInt));
      break;
    case "solr.IntPointField":
    case "solr.LongPointField":
      Collections.sort(valsAsStrings, Comparator.comparingLong(Long::parseLong));
      break;

    case "solr.TrieFloatField":
    case "solr.FloatPointField":
    case "solr.TrieDoubleField":
    case "solr.DoublePointField":
      Collections.sort(valsAsStrings, Comparator.comparingDouble(Double::parseDouble));
      break;

    case "solr.TrieDateField":
    case "solr.DatePointField":
    case "solr.StrField":
    case "solr.BoolField":
      Collections.sort(valsAsStrings);
      break;

    default:
      fail("Found no case for field " + name + " type " + type);
      break;
  }
}
 
Example 20
Source File: SearchImpl.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public List<SortField> guessSortTypes(String name) {
  FieldInfo finfo = IndexUtils.getFieldInfo(reader, name);
  if (finfo == null) {
    throw new LukeException("No such field: " + name, new IllegalArgumentException());
  }

  DocValuesType dvType = finfo.getDocValuesType();

  switch (dvType) {
    case NONE:
      return Collections.emptyList();

    case NUMERIC:
      return Arrays.stream(new SortField[]{
          new SortField(name, SortField.Type.INT),
          new SortField(name, SortField.Type.LONG),
          new SortField(name, SortField.Type.FLOAT),
          new SortField(name, SortField.Type.DOUBLE)
      }).collect(Collectors.toList());

    case SORTED_NUMERIC:
      return Arrays.stream(new SortField[]{
          new SortedNumericSortField(name, SortField.Type.INT),
          new SortedNumericSortField(name, SortField.Type.LONG),
          new SortedNumericSortField(name, SortField.Type.FLOAT),
          new SortedNumericSortField(name, SortField.Type.DOUBLE)
      }).collect(Collectors.toList());

    case SORTED:
      return Arrays.stream(new SortField[] {
          new SortField(name, SortField.Type.STRING),
          new SortField(name, SortField.Type.STRING_VAL)
      }).collect(Collectors.toList());

    case SORTED_SET:
      return Collections.singletonList(new SortedSetSortField(name, false));

    default:
      return Collections.singletonList(new SortField(name, SortField.Type.DOC));
  }

}