Java Code Examples for org.apache.lucene.index.FieldInfo#getDocValuesType()
The following examples show how to use
org.apache.lucene.index.FieldInfo#getDocValuesType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FacetFieldProcessorByHashDV.java From lucene-solr with Apache License 2.0 | 6 votes |
FacetFieldProcessorByHashDV(FacetContext fcontext, FacetField freq, SchemaField sf) { super(fcontext, freq, sf); if (freq.mincount == 0) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, getClass()+" doesn't support mincount=0"); } if (freq.prefix != null) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, getClass()+" doesn't support prefix"); // yet, but it could } FieldInfo fieldInfo = fcontext.searcher.getFieldInfos().fieldInfo(sf.getName()); if (fieldInfo != null && fieldInfo.getDocValuesType() != DocValuesType.NUMERIC && fieldInfo.getDocValuesType() != DocValuesType.SORTED && fieldInfo.getDocValuesType() != DocValuesType.SORTED_NUMERIC) { throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, getClass()+" only support single valued number/string with docValues"); } }
Example 2
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
public BinaryDocValues getTerms(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException { BinaryDocValues valuesIn = reader.getBinaryDocValues(field); if (valuesIn == null) { valuesIn = reader.getSortedDocValues(field); } if (valuesIn != null) { // Not cached here by FieldCacheImpl (cached instead // per-thread by SegmentReader): return valuesIn; } final FieldInfo info = reader.getFieldInfos().fieldInfo(field); if (info == null) { return DocValues.emptyBinary(); } else if (info.getDocValuesType() != DocValuesType.NONE) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } else if (info.getIndexOptions() == IndexOptions.NONE) { return DocValues.emptyBinary(); } BinaryDocValuesImpl impl = (BinaryDocValuesImpl) caches.get(BinaryDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio)); return impl.iterator(); }
Example 3
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException { SortedDocValues valuesIn = reader.getSortedDocValues(field); if (valuesIn != null) { // Not cached here by FieldCacheImpl (cached instead // per-thread by SegmentReader): return valuesIn; } else { final FieldInfo info = reader.getFieldInfos().fieldInfo(field); if (info == null) { return DocValues.emptySorted(); } else if (info.getDocValuesType() != DocValuesType.NONE) { // we don't try to build a sorted instance from numeric/binary doc // values because dedup can be very costly throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } else if (info.getIndexOptions() == IndexOptions.NONE) { return DocValues.emptySorted(); } SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio)); return impl.iterator(); } }
Example 4
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 6 votes |
@Override public Bits getDocsWithField(LeafReader reader, String field, Parser parser) throws IOException { final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); if (fieldInfo == null) { // field does not exist or has no value return new Bits.MatchNoBits(reader.maxDoc()); } if (fieldInfo.getDocValuesType() != DocValuesType.NONE) { // doc values case } else if (parser instanceof PointParser) { // points case } else { // postings case if (fieldInfo.getIndexOptions() == IndexOptions.NONE) { return new Bits.MatchNoBits(reader.maxDoc()); } } BitsEntry bitsEntry = (BitsEntry) caches.get(DocsWithFieldCache.class).get(reader, new CacheKey(field, parser)); return bitsEntry.bits; }
Example 5
Source File: VersionFieldUpgrader.java From Elasticsearch with Apache License 2.0 | 6 votes |
static CodecReader wrap(CodecReader reader) throws IOException { final FieldInfos fieldInfos = reader.getFieldInfos(); final FieldInfo versionInfo = fieldInfos.fieldInfo(VersionFieldMapper.NAME); if (versionInfo != null && versionInfo.getDocValuesType() != DocValuesType.NONE) { // the reader is a recent one, it has versions and they are stored // in a numeric doc values field return reader; } // The segment is an old one, look at the _uid field final Terms terms = reader.terms(UidFieldMapper.NAME); if (terms == null || !terms.hasPayloads()) { // The segment doesn't have an _uid field or doesn't have payloads // don't try to do anything clever. If any other segment has versions // all versions of this segment will be initialized to 0 return reader; } // convert _uid payloads -> _version docvalues return new VersionFieldUpgrader(reader); }
Example 6
Source File: LatLonDocValuesField.java From lucene-solr with Apache License 2.0 | 5 votes |
/** helper: checks a fieldinfo and throws exception if its definitely not a LatLonDocValuesField */ static void checkCompatible(FieldInfo fieldInfo) { // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + " but this type has docValuesType=" + TYPE.docValuesType() + ", is the field really a LatLonDocValuesField?"); } }
Example 7
Source File: SolrDocumentFetcher.java From lucene-solr with Apache License 2.0 | 5 votes |
private boolean canSubstituteDvForStored(FieldInfo fieldInfo, SchemaField schemaField) { if (!schemaField.hasDocValues() || !schemaField.stored()) return false; if (schemaField.multiValued()) return false; DocValuesType docValuesType = fieldInfo.getDocValuesType(); NumberType numberType = schemaField.getType().getNumberType(); // can not decode a numeric without knowing its numberType if (numberType == null && (docValuesType == DocValuesType.SORTED_NUMERIC || docValuesType == DocValuesType.NUMERIC)) { return false; } return true; }
Example 8
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 5 votes |
private BitsEntry createValueDocValues(LeafReader reader, String field) throws IOException { FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); DocValuesType dvType = fieldInfo.getDocValuesType(); DocIdSetIterator iterator; switch(dvType) { case NUMERIC: iterator = reader.getNumericDocValues(field); break; case BINARY: iterator = reader.getBinaryDocValues(field); break; case SORTED: iterator = reader.getSortedDocValues(field); break; case SORTED_NUMERIC: iterator = reader.getSortedNumericDocValues(field); break; case SORTED_SET: iterator = reader.getSortedSetDocValues(field); break; default: throw new AssertionError(); } FixedBitSet bits = new FixedBitSet(reader.maxDoc()); while (true) { int docID = iterator.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; } bits.set(docID); } return new BitsEntry(bits); }
Example 9
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override protected BitsEntry createValue(LeafReader reader, CacheKey key) throws IOException { final String field = key.field; final Parser parser = (Parser) key.custom; FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); if (fieldInfo.getDocValuesType() != DocValuesType.NONE) { return createValueDocValues(reader, field); } else if (parser instanceof PointParser) { return createValuePoints(reader, field); } else { return createValuePostings(reader, field); } }
Example 10
Source File: XYDocValuesField.java From lucene-solr with Apache License 2.0 | 5 votes |
/** helper: checks a fieldinfo and throws exception if its definitely not a XYDocValuesField */ static void checkCompatible(FieldInfo fieldInfo) { // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + " but this type has docValuesType=" + TYPE.docValuesType() + ", is the field really a XYDocValuesField?"); } }
Example 11
Source File: UninvertDocValuesMergePolicyFactory.java From lucene-solr with Apache License 2.0 | 5 votes |
private UninvertingReader.Type getUninversionType(FieldInfo fi) { SchemaField sf = schema.getFieldOrNull(fi.name); if (null != sf && sf.hasDocValues() && fi.getDocValuesType() == DocValuesType.NONE && fi.getIndexOptions() != IndexOptions.NONE) { return sf.getType().getUninversionType(sf); } else { return null; } }
Example 12
Source File: PerFieldDocValuesFormat.java From lucene-solr with Apache License 2.0 | 5 votes |
public FieldsReader(final SegmentReadState readState) throws IOException { // Init each unique format: boolean success = false; try { // Read field name -> format name for (FieldInfo fi : readState.fieldInfos) { if (fi.getDocValuesType() != DocValuesType.NONE) { final String fieldName = fi.name; final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY); if (formatName != null) { // null formatName means the field is in fieldInfos, but has no docvalues! final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY); if (suffix == null) { throw new IllegalStateException("missing attribute: " + PER_FIELD_SUFFIX_KEY + " for field: " + fieldName); } DocValuesFormat format = DocValuesFormat.forName(formatName); String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix)); if (!formats.containsKey(segmentSuffix)) { formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); } fields.put(fieldName, formats.get(segmentSuffix)); } } } success = true; } finally { if (!success) { IOUtils.closeWhileHandlingException(formats.values()); } } }
Example 13
Source File: PerFieldDocValuesFormat.java From lucene-solr with Apache License 2.0 | 5 votes |
@Override public void merge(MergeState mergeState) throws IOException { Map<DocValuesConsumer, Collection<String>> consumersToField = new IdentityHashMap<>(); // Group each consumer by the fields it handles for (FieldInfo fi : mergeState.mergeFieldInfos) { if (fi.getDocValuesType() == DocValuesType.NONE) { continue; } // merge should ignore current format for the fields being merged DocValuesConsumer consumer = getInstance(fi, true); Collection<String> fieldsForConsumer = consumersToField.get(consumer); if (fieldsForConsumer == null) { fieldsForConsumer = new ArrayList<>(); consumersToField.put(consumer, fieldsForConsumer); } fieldsForConsumer.add(fi.name); } // Delegate the merge to the appropriate consumer PerFieldMergeState pfMergeState = new PerFieldMergeState(mergeState); try { for (Map.Entry<DocValuesConsumer, Collection<String>> e : consumersToField.entrySet()) { e.getKey().merge(pfMergeState.apply(e.getValue())); } } finally { pfMergeState.reset(); } }
Example 14
Source File: PerFieldMergeState.java From lucene-solr with Apache License 2.0 | 5 votes |
FilterFieldInfos(FieldInfos src, Collection<String> filterFields) { // Copy all the input FieldInfo objects since the field numbering must be kept consistent super(toArray(src)); boolean hasVectors = false; boolean hasProx = false; boolean hasPayloads = false; boolean hasOffsets = false; boolean hasFreq = false; boolean hasNorms = false; boolean hasDocValues = false; boolean hasPointValues = false; this.filteredNames = new HashSet<>(filterFields); this.filtered = new ArrayList<>(filterFields.size()); for (FieldInfo fi : src) { if (this.filteredNames.contains(fi.name)) { this.filtered.add(fi); hasVectors |= fi.hasVectors(); hasProx |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; hasFreq |= fi.getIndexOptions() != IndexOptions.DOCS; hasOffsets |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; hasNorms |= fi.hasNorms(); hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE; hasPayloads |= fi.hasPayloads(); hasPointValues |= (fi.getPointDimensionCount() != 0); } } this.filteredHasVectors = hasVectors; this.filteredHasProx = hasProx; this.filteredHasPayloads = hasPayloads; this.filteredHasOffsets = hasOffsets; this.filteredHasFreq = hasFreq; this.filteredHasNorms = hasNorms; this.filteredHasDocValues = hasDocValues; this.filteredHasPointValues = hasPointValues; }
Example 15
Source File: DocValuesConsumer.java From lucene-solr with Apache License 2.0 | 5 votes |
/** Merges in the fields from the readers in * <code>mergeState</code>. The default implementation * calls {@link #mergeNumericField}, {@link #mergeBinaryField}, * {@link #mergeSortedField}, {@link #mergeSortedSetField}, * or {@link #mergeSortedNumericField} for each field, * depending on its type. * Implementations can override this method * for more sophisticated merging (bulk-byte copying, etc). */ public void merge(MergeState mergeState) throws IOException { for(DocValuesProducer docValuesProducer : mergeState.docValuesProducers) { if (docValuesProducer != null) { docValuesProducer.checkIntegrity(); } } for (FieldInfo mergeFieldInfo : mergeState.mergeFieldInfos) { DocValuesType type = mergeFieldInfo.getDocValuesType(); if (type != DocValuesType.NONE) { if (type == DocValuesType.NUMERIC) { mergeNumericField(mergeFieldInfo, mergeState); } else if (type == DocValuesType.BINARY) { mergeBinaryField(mergeFieldInfo, mergeState); } else if (type == DocValuesType.SORTED) { mergeSortedField(mergeFieldInfo, mergeState); } else if (type == DocValuesType.SORTED_SET) { mergeSortedSetField(mergeFieldInfo, mergeState); } else if (type == DocValuesType.SORTED_NUMERIC) { mergeSortedNumericField(mergeFieldInfo, mergeState); } else { throw new AssertionError("type=" + type); } } } }
Example 16
Source File: DocumentField.java From lucene-solr with Apache License 2.0 | 5 votes |
static DocumentField of(FieldInfo finfo, IndexableField field, IndexReader reader, int docId) throws IOException { Objects.requireNonNull(finfo); Objects.requireNonNull(reader); DocumentField dfield = new DocumentField(); dfield.name = finfo.name; dfield.idxOptions = finfo.getIndexOptions(); dfield.hasTermVectors = finfo.hasVectors(); dfield.hasPayloads = finfo.hasPayloads(); dfield.hasNorms = finfo.hasNorms(); if (finfo.hasNorms()) { NumericDocValues norms = MultiDocValues.getNormValues(reader, finfo.name); if (norms.advanceExact(docId)) { dfield.norm = norms.longValue(); } } dfield.dvType = finfo.getDocValuesType(); dfield.pointDimensionCount = finfo.getPointDimensionCount(); dfield.pointNumBytes = finfo.getPointNumBytes(); if (field != null) { dfield.isStored = field.fieldType().stored(); dfield.stringValue = field.stringValue(); if (field.binaryValue() != null) { dfield.binaryValue = BytesRef.deepCopyOf(field.binaryValue()); } dfield.numericValue = field.numericValue(); } return dfield; }
Example 17
Source File: Geo3DDocValuesField.java From lucene-solr with Apache License 2.0 | 5 votes |
/** helper: checks a fieldinfo and throws exception if its definitely not a Geo3DDocValuesField */ static void checkCompatible(FieldInfo fieldInfo) { // dv properties could be "unset", if you e.g. used only StoredField with this same name in the segment. if (fieldInfo.getDocValuesType() != DocValuesType.NONE && fieldInfo.getDocValuesType() != TYPE.docValuesType()) { throw new IllegalArgumentException("field=\"" + fieldInfo.name + "\" was indexed with docValuesType=" + fieldInfo.getDocValuesType() + " but this type has docValuesType=" + TYPE.docValuesType() + ", is the field really a Geo3DDocValuesField?"); } }
Example 18
Source File: FieldCacheImpl.java From lucene-solr with Apache License 2.0 | 4 votes |
@Override public NumericDocValues getNumerics(LeafReader reader, String field, Parser parser) throws IOException { if (parser == null) { throw new NullPointerException(); } final NumericDocValues valuesIn = reader.getNumericDocValues(field); if (valuesIn != null) { return valuesIn; } else { final FieldInfo info = reader.getFieldInfos().fieldInfo(field); if (info == null) { return DocValues.emptyNumeric(); } else if (info.getDocValuesType() != DocValuesType.NONE) { throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType()); } if (parser instanceof PointParser) { // points case // no points in this segment if (info.getPointDimensionCount() == 0) { return DocValues.emptyNumeric(); } if (info.getPointDimensionCount() != 1) { throw new IllegalStateException("Type mismatch: " + field + " was indexed with dimensions=" + info.getPointDimensionCount()); } PointValues values = reader.getPointValues(field); // no actual points for this field (e.g. all points deleted) if (values == null || values.size() == 0) { return DocValues.emptyNumeric(); } // not single-valued if (values.size() != values.getDocCount()) { throw new IllegalStateException("Type mismatch: " + field + " was indexed with multiple values, numValues=" + values.size() + ",numDocs=" + values.getDocCount()); } } else { // postings case // not indexed if (info.getIndexOptions() == IndexOptions.NONE) { return DocValues.emptyNumeric(); } } return ((LongsFromArray) caches.get(Long.TYPE).get(reader, new CacheKey(field, parser))).iterator(); } }
Example 19
Source File: TestRetrieveFieldsOptimizer.java From lucene-solr with Apache License 2.0 | 4 votes |
void expectedValsAsStrings(final FieldInfo info, List<String> valsAsStrings) { if (schemaField.stored() || schemaField.multiValued() == false) { return ; } switch (info.getDocValuesType()) { case NONE: // These three types are single values, just return. case NUMERIC: case BINARY: // here for completeness, really doesn't make sense. return; case SORTED_NUMERIC: // Can have multiple, identical values. This was a surprise to me. break; case SORTED_SET: // Obey set semantics. case SORTED: Set<String> uniq = new TreeSet<>(valsAsStrings); valsAsStrings.clear(); valsAsStrings.addAll(uniq); break; } // Now order them if string-based comparison isn't reasonable switch (testFieldType.getSolrTypeClass()) { case "solr.TrieIntField": case "solr.TrieLongField": Collections.sort(valsAsStrings, Comparator.comparingInt(Integer::parseInt)); break; case "solr.IntPointField": case "solr.LongPointField": Collections.sort(valsAsStrings, Comparator.comparingLong(Long::parseLong)); break; case "solr.TrieFloatField": case "solr.FloatPointField": case "solr.TrieDoubleField": case "solr.DoublePointField": Collections.sort(valsAsStrings, Comparator.comparingDouble(Double::parseDouble)); break; case "solr.TrieDateField": case "solr.DatePointField": case "solr.StrField": case "solr.BoolField": Collections.sort(valsAsStrings); break; default: fail("Found no case for field " + name + " type " + type); break; } }
Example 20
Source File: SearchImpl.java From lucene-solr with Apache License 2.0 | 4 votes |
@Override public List<SortField> guessSortTypes(String name) { FieldInfo finfo = IndexUtils.getFieldInfo(reader, name); if (finfo == null) { throw new LukeException("No such field: " + name, new IllegalArgumentException()); } DocValuesType dvType = finfo.getDocValuesType(); switch (dvType) { case NONE: return Collections.emptyList(); case NUMERIC: return Arrays.stream(new SortField[]{ new SortField(name, SortField.Type.INT), new SortField(name, SortField.Type.LONG), new SortField(name, SortField.Type.FLOAT), new SortField(name, SortField.Type.DOUBLE) }).collect(Collectors.toList()); case SORTED_NUMERIC: return Arrays.stream(new SortField[]{ new SortedNumericSortField(name, SortField.Type.INT), new SortedNumericSortField(name, SortField.Type.LONG), new SortedNumericSortField(name, SortField.Type.FLOAT), new SortedNumericSortField(name, SortField.Type.DOUBLE) }).collect(Collectors.toList()); case SORTED: return Arrays.stream(new SortField[] { new SortField(name, SortField.Type.STRING), new SortField(name, SortField.Type.STRING_VAL) }).collect(Collectors.toList()); case SORTED_SET: return Collections.singletonList(new SortedSetSortField(name, false)); default: return Collections.singletonList(new SortField(name, SortField.Type.DOC)); } }