org.apache.lucene.codecs.DocValuesProducer Java Examples

The following examples show how to use org.apache.lucene.codecs.DocValuesProducer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SegmentDocValues.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private RefCount<DocValuesProducer> newDocValuesProducer(SegmentCommitInfo si, Directory dir, final Long gen, FieldInfos infos) throws IOException {
  Directory dvDir = dir;
  String segmentSuffix = "";
  if (gen.longValue() != -1) {
    dvDir = si.info.dir; // gen'd files are written outside CFS, so use SegInfo directory
    segmentSuffix = Long.toString(gen.longValue(), Character.MAX_RADIX);
  }

  // set SegmentReadState to list only the fields that are relevant to that gen
  SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, IOContext.READ, segmentSuffix);
  DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat();
  return new RefCount<DocValuesProducer>(dvFormat.fieldsProducer(srs)) {
    @SuppressWarnings("synthetic-access")
    @Override
    protected void release() throws IOException {
      object.close();
      synchronized (SegmentDocValues.this) {
        genDVProducers.remove(gen);
      }
    }
  };
}
 
Example #2
Source File: SegmentReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * init most recent DocValues for the current commit
 */
private DocValuesProducer initDocValuesProducer() throws IOException {

  if (fieldInfos.hasDocValues() == false) {
    return null;
  } else {
    Directory dir;
    if (core.cfsReader != null) {
      dir = core.cfsReader;
    } else {
      dir = si.info.dir;
    }
    if (si.hasFieldUpdates()) {
      return new SegmentDocValuesProducer(si, dir, core.coreFieldInfos, fieldInfos, segDocValues);
    } else {
      // simple case, no DocValues updates
      return segDocValues.getDocValuesProducer(-1L, si, dir, fieldInfos);
    }
  }
}
 
Example #3
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);

  long valueCount = 0;
  int lastDocID = -1;
  while (true) {
    int docID = values.nextDoc();
    if (docID == NO_MORE_DOCS) {
      break;
    }
    assert values.docID() > lastDocID;
    lastDocID = values.docID();
    int count = values.docValueCount();
    assert count > 0;
    valueCount += count;
    long previous = Long.MIN_VALUE;
    for (int i = 0; i < count; i++) {
      long nextValue = values.nextValue();
      assert nextValue >= previous;
      previous = nextValue;
    }
  }
  in.addSortedNumericField(field, valuesProducer);
}
 
Example #4
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  BinaryDocValues values = valuesProducer.getBinary(field);
  
  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    BytesRef value = values.binaryValue();
    assert value.isValid();
  }

  in.addBinaryField(field, valuesProducer);
}
 
Example #5
Source File: VersionFieldUpgrader.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public DocValuesProducer getDocValuesReader() {
    DocValuesProducer producer = in.getDocValuesReader();
    // TODO: move this nullness stuff out
    if (producer == null) {
        producer = FilterDocValuesProducer.EMPTY;
    }
    return new UninvertedVersions(producer, this);
}
 
Example #6
Source File: MergeReaderWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
MergeReaderWrapper(CodecReader in) throws IOException {
  this.in = in;
  
  FieldsProducer fields = in.getPostingsReader();
  if (fields != null) {
    fields = fields.getMergeInstance();
  }
  this.fields = fields;
  
  NormsProducer norms = in.getNormsReader();
  if (norms != null) {
    norms = norms.getMergeInstance();
  }
  this.norms = norms;
  
  DocValuesProducer docValues = in.getDocValuesReader();
  if (docValues != null) {
    docValues = docValues.getMergeInstance();
  }
  this.docValues = docValues;
  
  StoredFieldsReader store = in.getFieldsReader();
  if (store != null) {
    store = store.getMergeInstance();
  }
  this.store = store;
  
  TermVectorsReader vectors = in.getTermVectorsReader();
  if (vectors != null) {
    vectors = vectors.getMergeInstance();
  }
  this.vectors = vectors;
}
 
Example #7
Source File: CrankyDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedSetField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedSetField()");
  }
  delegate.addSortedSetField(field, valuesProducer);
}
 
Example #8
Source File: CrankyDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedNumericField()");
  }
  delegate.addSortedNumericField(field, valuesProducer);
}
 
Example #9
Source File: CrankyDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addSortedField()");
  }
  delegate.addSortedField(field, valuesProducer);
}
 
Example #10
Source File: CrankyDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addBinaryField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addBinaryField()");
  }
  delegate.addBinaryField(field, valuesProducer);
}
 
Example #11
Source File: CrankyDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  if (random.nextInt(100) == 0) {
    throw new IOException("Fake IOException from DocValuesConsumer.addNumericField()");
  }
  delegate.addNumericField(field, valuesProducer);
}
 
Example #12
Source File: SegmentDocValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Returns the {@link DocValuesProducer} for the given generation. */
synchronized DocValuesProducer getDocValuesProducer(long gen, SegmentCommitInfo si, Directory dir, FieldInfos infos) throws IOException {
  RefCount<DocValuesProducer> dvp = genDVProducers.get(gen);
  if (dvp == null) {
    dvp = newDocValuesProducer(si, dir, gen, infos);
    assert dvp != null;
    genDVProducers.put(gen, dvp);
  } else {
    dvp.incRef();
  }
  return dvp.get();
}
 
Example #13
Source File: CheckIndex.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void checkDocValues(FieldInfo fi, DocValuesProducer dvReader, int maxDoc, PrintStream infoStream, DocValuesStatus status) throws Exception {
  switch(fi.getDocValuesType()) {
    case SORTED:
      status.totalSortedFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSorted);
      checkBinaryDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
      checkSortedDocValues(fi.name, maxDoc, dvReader.getSorted(fi), dvReader.getSorted(fi));
      break;
    case SORTED_NUMERIC:
      status.totalSortedNumericFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSortedNumeric);
      checkSortedNumericDocValues(fi.name, maxDoc, dvReader.getSortedNumeric(fi), dvReader.getSortedNumeric(fi));
      break;
    case SORTED_SET:
      status.totalSortedSetFields++;
      checkDVIterator(fi, maxDoc, dvReader::getSortedSet);
      checkSortedSetDocValues(fi.name, maxDoc, dvReader.getSortedSet(fi), dvReader.getSortedSet(fi));
      break;
    case BINARY:
      status.totalBinaryFields++;
      checkDVIterator(fi, maxDoc, dvReader::getBinary);
      checkBinaryDocValues(fi.name, maxDoc, dvReader.getBinary(fi), dvReader.getBinary(fi));
      break;
    case NUMERIC:
      status.totalNumericFields++;
      checkDVIterator(fi, maxDoc, dvReader::getNumeric);
      checkNumericDocValues(fi.name, dvReader.getNumeric(fi), dvReader.getNumeric(fi));
      break;
    default:
      throw new AssertionError();
  }
}
 
Example #14
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
AssertingDocValuesProducer(DocValuesProducer in, int maxDoc, boolean merging) {
  this.in = in;
  this.maxDoc = maxDoc;
  this.merging = merging;
  this.creationThread = Thread.currentThread();
  // do a few simple checks on init
  assert toString() != null;
  assert ramBytesUsed() >= 0;
  assert getChildResources() != null;
}
 
Example #15
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  SortedDocValues values = valuesProducer.getSorted(field);

  int valueCount = values.getValueCount();
  assert valueCount <= maxDoc;
  BytesRef lastValue = null;
  for (int ord=0;ord<valueCount;ord++) {
    BytesRef b = values.lookupOrd(ord);
    assert b != null;
    assert b.isValid();
    if (ord > 0) {
      assert b.compareTo(lastValue) > 0;
    }
    lastValue = BytesRef.deepCopyOf(b);
  }
  
  FixedBitSet seenOrds = new FixedBitSet(valueCount);
  
  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    int ord = values.ordValue();
    assert ord >= 0 && ord < valueCount;
    seenOrds.set(ord);
  }
  
  assert seenOrds.cardinality() == valueCount;
  in.addSortedField(field, valuesProducer);
}
 
Example #16
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  NumericDocValues values = valuesProducer.getNumeric(field);

  int docID;
  int lastDocID = -1;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    assert docID >= 0 && docID < maxDoc;
    assert docID > lastDocID;
    lastDocID = docID;
    long value = values.longValue();
  }
  
  in.addNumericField(field, valuesProducer);
}
 
Example #17
Source File: AssertingDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
  assert state.fieldInfos.hasDocValues();
  DocValuesProducer producer = in.fieldsProducer(state);
  assert producer != null;
  return new AssertingDocValuesProducer(producer, state.segmentInfo.maxDoc(), false);
}
 
Example #18
Source File: Lucene80DocValuesConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void addSortedNumericField(FieldInfo field, DocValuesProducer valuesProducer) throws IOException {
  meta.writeInt(field.number);
  meta.writeByte(Lucene80DocValuesFormat.SORTED_NUMERIC);

  long[] stats = writeValues(field, valuesProducer);
  int numDocsWithField = Math.toIntExact(stats[0]);
  long numValues = stats[1];
  assert numValues >= numDocsWithField;

  meta.writeInt(numDocsWithField);
  if (numValues > numDocsWithField) {
    long start = data.getFilePointer();
    meta.writeLong(start);
    meta.writeVInt(DIRECT_MONOTONIC_BLOCK_SHIFT);

    final DirectMonotonicWriter addressesWriter = DirectMonotonicWriter.getInstance(meta, data, numDocsWithField + 1L, DIRECT_MONOTONIC_BLOCK_SHIFT);
    long addr = 0;
    addressesWriter.add(addr);
    SortedNumericDocValues values = valuesProducer.getSortedNumeric(field);
    for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
      addr += values.docValueCount();
      addressesWriter.add(addr);
    }
    addressesWriter.finish();
    meta.writeLong(data.getFilePointer() - start);
  }
}
 
Example #19
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public long ramBytesUsed() {
  long size = 0;
  for (Map.Entry<String,DocValuesProducer> entry : formats.entrySet()) {
    size += (entry.getKey().length() * Character.BYTES) + entry.getValue().ramBytesUsed();
  }
  return size;
}
 
Example #20
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public DocValuesProducer getDocValuesReader() {
    DocValuesProducer docValuesReader = super.getDocValuesReader();
    return new FilterDocValuesProducer(docValuesReader) {
        @Override
        public NumericDocValues getNumeric(FieldInfo field) throws IOException {
            NumericDocValues numeric = super.getNumeric(field);
            if (recoverySourceField.equals(field.name)) {
                assert numeric != null : recoverySourceField + " must have numeric DV but was null";
                final DocIdSetIterator intersection;
                if (recoverySourceToKeep == null) {
                    // we can't return null here lucenes DocIdMerger expects an instance
                    intersection = DocIdSetIterator.empty();
                } else {
                    intersection = ConjunctionDISI.intersectIterators(Arrays.asList(numeric,
                                                                                    new BitSetIterator(recoverySourceToKeep, recoverySourceToKeep.length())));
                }
                return new FilterNumericDocValues(numeric) {
                    @Override
                    public int nextDoc() throws IOException {
                        return intersection.nextDoc();
                    }

                    @Override
                    public int advance(int target) {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public boolean advanceExact(int target) {
                        throw new UnsupportedOperationException();
                    }
                };

            }
            return numeric;
        }
    };
}
 
Example #21
Source File: CheckIndex.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/**
 * Test docvalues.
 * @lucene.experimental
 */
public static Status.DocValuesStatus testDocValues(CodecReader reader,
                                                   PrintStream infoStream,
                                                   boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.DocValuesStatus status = new Status.DocValuesStatus();
  try {
    if (infoStream != null) {
      infoStream.print("    test: docvalues...........");
    }
    DocValuesProducer dvReader = reader.getDocValuesReader();
    if (dvReader != null) {
      dvReader = dvReader.getMergeInstance();
    }
    for (FieldInfo fieldInfo : reader.getFieldInfos()) {
      if (fieldInfo.getDocValuesType() != DocValuesType.NONE) {
        status.totalValueFields++;
        checkDocValues(fieldInfo, dvReader, reader.maxDoc(), infoStream, status);
      }
    }

    msg(infoStream, String.format(Locale.ROOT,
                                  "OK [%d docvalues fields; %d BINARY; %d NUMERIC; %d SORTED; %d SORTED_NUMERIC; %d SORTED_SET] [took %.3f sec]",
                                  status.totalValueFields,
                                  status.totalBinaryFields,
                                  status.totalNumericFields,
                                  status.totalSortedFields,
                                  status.totalSortedNumericFields,
                                  status.totalSortedSetFields,
                                  nsToSec(System.nanoTime()-startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }
  return status;
}
 
Example #22
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesProducer fieldsProducer(SegmentReadState state) throws IOException {
  return new FieldsReader(state);
}
 
Example #23
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesProducer getMergeInstance() {
  return new FieldsReader(this);
}
 
Example #24
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void checkIntegrity() throws IOException {
  for (DocValuesProducer format : formats.values()) {
    format.checkIntegrity();
  }
}
 
Example #25
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSortedSet(field);
}
 
Example #26
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSortedNumeric(field);
}
 
Example #27
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 4 votes vote down vote up
FilterDocValuesProducer(DocValuesProducer in) {
    this.in = in;
}
 
Example #28
Source File: PerFieldDocValuesFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public SortedDocValues getSorted(FieldInfo field) throws IOException {
  DocValuesProducer producer = fields.get(field.name);
  return producer == null ? null : producer.getSorted(field);
}
 
Example #29
Source File: UninvertDocValuesMergePolicyFactory.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesProducer getDocValuesReader() {
  return docValuesProducer;
}
 
Example #30
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public OrdScoreCollector(int maxDoc,
                         int segments,
                         DocValuesProducer collapseValuesProducer,
                         int nullPolicy,
                         IntIntHashMap boostDocsMap,
                         IndexSearcher searcher) throws IOException {
  this.maxDoc = maxDoc;
  this.contexts = new LeafReaderContext[segments];
  List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
  for(int i=0; i<con.size(); i++) {
    contexts[i] = con.get(i);
  }

  this.collapsedSet = new FixedBitSet(maxDoc);
  this.collapseValuesProducer = collapseValuesProducer;
  this.collapseValues = collapseValuesProducer.getSorted(null);

  int valueCount = collapseValues.getValueCount();
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  this.ords = new IntIntDynamicMap(valueCount, -1);
  this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
  this.nullPolicy = nullPolicy;
  if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
    nullScores = new FloatArrayList();
  }

  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostOrds = new IntArrayList();
    this.boostDocs = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
  }
}