org.apache.lucene.codecs.StoredFieldsReader Java Examples

The following examples show how to use org.apache.lucene.codecs.StoredFieldsReader. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 6 votes vote down vote up
@Override
public StoredFieldsReader getFieldsReader() {
    StoredFieldsReader fieldsReader = super.getFieldsReader();
    return new FilterStoredFieldsReader(fieldsReader) {
        @Override
        public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
            if (recoverySourceToKeep != null && recoverySourceToKeep.get(docID)) {
                super.visitDocument(docID, visitor);
            } else {
                super.visitDocument(docID, new FilterStoredFieldVisitor(visitor) {
                    @Override
                    public Status needsField(FieldInfo fieldInfo) throws IOException {
                        if (recoverySourceField.equals(fieldInfo.name)) {
                            return Status.NO;
                        }
                        return super.needsField(fieldInfo);
                    }
                });
            }
        }
    };
}
 
Example #2
Source File: SimpleTextStoredFieldsReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public StoredFieldsReader clone() {
  if (in == null) {
    throw new AlreadyClosedException("this FieldsReader is closed");
  }
  return new SimpleTextStoredFieldsReader(offsets, in.clone(), fieldInfos);
}
 
Example #3
Source File: AssertingStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
AssertingStoredFieldsReader(StoredFieldsReader in, int maxDoc, boolean merging) {
this.in = in;
this.maxDoc = maxDoc;
this.merging = merging;
this.creationThread = Thread.currentThread();
// do a few simple checks on init
 
Example #4
Source File: IndexSizeEstimator.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void estimateStoredFields(Map<String, Object> result) throws IOException {
  log.info("- estimating stored fields...");
  Map<String, Map<String, Object>> stats = new HashMap<>();
  for (LeafReaderContext context : reader.leaves()) {
    LeafReader leafReader = context.reader();
    EstimatingVisitor visitor = new EstimatingVisitor(stats, topN, maxLength, samplingStep);
    Bits liveDocs = leafReader.getLiveDocs();
    if (leafReader instanceof CodecReader) {
      CodecReader codecReader = (CodecReader)leafReader;
      StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
      // this instance may be faster for a full sequential pass
      StoredFieldsReader mergeInstance = storedFieldsReader.getMergeInstance();
      for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        mergeInstance.visitDocument(docId, visitor);
      }
      if (mergeInstance != storedFieldsReader) {
        mergeInstance.close();
      }
    } else {
      for (int docId = 0; docId < leafReader.maxDoc(); docId += samplingStep) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        leafReader.document(docId, visitor);
      }
    }
  }
  result.put(STORED_FIELDS, stats);
}
 
Example #5
Source File: Lucene50StoredFieldsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  String value = si.getAttribute(MODE_KEY);
  if (value == null) {
    throw new IllegalStateException("missing value for " + MODE_KEY + " for segment: " + si.name);
  }
  Mode mode = Mode.valueOf(value);
  return impl(mode).fieldsReader(directory, si, fn, context);
}
 
Example #6
Source File: SlowCodecReaderWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static StoredFieldsReader readerToStoredFieldsReader(final LeafReader reader) {
  return new StoredFieldsReader() {
    @Override
    public void visitDocument(int docID, StoredFieldVisitor visitor) throws IOException {
      reader.document(docID, visitor);
    }

    @Override
    public StoredFieldsReader clone() {
      return readerToStoredFieldsReader(reader);
    }

    @Override
    public void checkIntegrity() throws IOException {
      // We already checkIntegrity the entire reader up front
    }

    @Override
    public void close() {
    }

    @Override
    public long ramBytesUsed() {
      return 0;
    }
  };
}
 
Example #7
Source File: MergeReaderWrapper.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
MergeReaderWrapper(CodecReader in) throws IOException {
  this.in = in;
  
  FieldsProducer fields = in.getPostingsReader();
  if (fields != null) {
    fields = fields.getMergeInstance();
  }
  this.fields = fields;
  
  NormsProducer norms = in.getNormsReader();
  if (norms != null) {
    norms = norms.getMergeInstance();
  }
  this.norms = norms;
  
  DocValuesProducer docValues = in.getDocValuesReader();
  if (docValues != null) {
    docValues = docValues.getMergeInstance();
  }
  this.docValues = docValues;
  
  StoredFieldsReader store = in.getFieldsReader();
  if (store != null) {
    store = store.getMergeInstance();
  }
  this.store = store;
  
  TermVectorsReader vectors = in.getTermVectorsReader();
  if (vectors != null) {
    vectors = vectors.getMergeInstance();
  }
  this.vectors = vectors;
}
 
Example #8
Source File: SortingStoredFieldsConsumer.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
void flush(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
  super.flush(state, sortMap);
  if (sortMap == null) {
    // we're lucky the index is already sorted, just rename the temporary file and return
    for (Map.Entry<String, String> entry : tmpDirectory.getTemporaryFiles().entrySet()) {
      tmpDirectory.rename(entry.getValue(), entry.getKey());
    }
    return;
  }
  StoredFieldsReader reader = docWriter.codec.storedFieldsFormat()
      .fieldsReader(tmpDirectory, state.segmentInfo, state.fieldInfos, IOContext.DEFAULT);
  StoredFieldsReader mergeReader = reader.getMergeInstance();
  StoredFieldsWriter sortWriter = docWriter.codec.storedFieldsFormat()
      .fieldsWriter(state.directory, state.segmentInfo, IOContext.DEFAULT);
  try {
    reader.checkIntegrity();
    CopyVisitor visitor = new CopyVisitor(sortWriter);
    for (int docID = 0; docID < state.segmentInfo.maxDoc(); docID++) {
      sortWriter.startDocument();
      mergeReader.visitDocument(sortMap.newToOld(docID), visitor);
      sortWriter.finishDocument();
    }
    sortWriter.finish(state.fieldInfos, state.segmentInfo.maxDoc());
  } finally {
    IOUtils.close(reader, sortWriter);
    IOUtils.deleteFiles(tmpDirectory,
        tmpDirectory.getTemporaryFiles().values());
  }
}
 
Example #9
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader clone() {
    return fieldsReader.clone();
}
 
Example #10
Source File: RecoverySourcePruneMergePolicy.java    From crate with Apache License 2.0 4 votes vote down vote up
FilterStoredFieldsReader(StoredFieldsReader fieldsReader) {
    this.fieldsReader = fieldsReader;
}
 
Example #11
Source File: Blur022StoredFieldsFormat.java    From incubator-retired-blur with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context)
    throws IOException {
  CompressionMode compressionMode = new CachedCompressionMode(getCompressionMode(si));
  return new CompressingStoredFieldsReader(directory, si, SEGMENT_SUFFIX, fn, context, FORMAT_NAME, compressionMode);
}
 
Example #12
Source File: IndexSizeEstimatorTest.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Test
public void testEstimator() throws Exception {
  JettySolrRunner jetty = cluster.getRandomJetty(random());
  String randomCoreName = jetty.getCoreContainer().getAllCoreNames().iterator().next();
  SolrCore core = jetty.getCoreContainer().getCore(randomCoreName);
  RefCounted<SolrIndexSearcher> searcherRef = core.getSearcher();
  try {
    SolrIndexSearcher searcher = searcherRef.get();
    // limit the max length
    IndexSizeEstimator estimator = new IndexSizeEstimator(searcher.getRawReader(), 20, 50, true, true);
    IndexSizeEstimator.Estimate estimate = estimator.estimate();
    Map<String, Long> fieldsBySize = estimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", fieldsBySize.isEmpty());
    assertEquals(fieldsBySize.toString(), fields.size(), fieldsBySize.size());
    fieldsBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Long> typesBySize = estimate.getTypesBySize();
    assertFalse("empty typesBySize", typesBySize.isEmpty());
    assertTrue("expected at least 8 types: " + typesBySize.toString(), typesBySize.size() >= 8);
    typesBySize.forEach((k, v) -> assertTrue("unexpected size of " + k + ": " + v, v > 0));
    Map<String, Object> summary = estimate.getSummary();
    assertNotNull("summary", summary);
    assertFalse("empty summary", summary.isEmpty());
    assertEquals(summary.keySet().toString(), fields.size(), summary.keySet().size());
    Map<String, Object> details = estimate.getDetails();
    assertNotNull("details", details);
    assertFalse("empty details", details.isEmpty());
    // by type
    assertEquals(details.keySet().toString(), 6, details.keySet().size());

    // check sampling
    estimator.setSamplingThreshold(searcher.getRawReader().maxDoc() / 2);
    IndexSizeEstimator.Estimate sampledEstimate = estimator.estimate();
    Map<String, Long> sampledFieldsBySize = sampledEstimate.getFieldsBySize();
    assertFalse("empty fieldsBySize", sampledFieldsBySize.isEmpty());
    // verify that the sampled values are within 50% of the original values
    fieldsBySize.forEach((field, size) -> {
      Long sampledSize = sampledFieldsBySize.get(field);
      assertNotNull("sampled size for " + field + " is missing in " + sampledFieldsBySize, sampledSize);
      double delta = (double) size * 0.5;
      assertEquals("sampled size of " + field + " is wildly off", (double)size, (double)sampledSize, delta);
    });
    // verify the reader is still usable - SOLR-13694
    IndexReader reader = searcher.getRawReader();
    for (LeafReaderContext context : reader.leaves()) {
      LeafReader leafReader = context.reader();
      assertTrue("unexpected LeafReader class: " + leafReader.getClass().getName(), leafReader instanceof CodecReader);
      Bits liveDocs = leafReader.getLiveDocs();
      CodecReader codecReader = (CodecReader) leafReader;
      StoredFieldsReader storedFieldsReader = codecReader.getFieldsReader();
      StoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      assertNotNull(storedFieldsReader);
      for (int docId = 0; docId < leafReader.maxDoc(); docId++) {
        if (liveDocs != null && !liveDocs.get(docId)) {
          continue;
        }
        storedFieldsReader.visitDocument(docId, visitor);
      }
    }
  } finally {
    searcherRef.decref();
    core.close();
  }
}
 
Example #13
Source File: SegmentCoreReaders.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
protected StoredFieldsReader initialValue() {
  return fieldsReaderOrig.clone();
}
 
Example #14
Source File: SlowCodecReaderWrapper.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/**
 * Returns a {@code CodecReader} view of reader. 
 * <p>
 * If {@code reader} is already a {@code CodecReader}, it is returned
 * directly. Otherwise, a (slow) view is returned.
 */
public static CodecReader wrap(final LeafReader reader) throws IOException {
  if (reader instanceof CodecReader) {
    return (CodecReader) reader;
  } else {
    // simulate it slowly, over the leafReader api:
    reader.checkIntegrity();
    return new CodecReader() {

      @Override
      public TermVectorsReader getTermVectorsReader() {
        reader.ensureOpen();
        return readerToTermVectorsReader(reader);
      }

      @Override
      public StoredFieldsReader getFieldsReader() {
        reader.ensureOpen();
        return readerToStoredFieldsReader(reader);
      }

      @Override
      public NormsProducer getNormsReader() {
        reader.ensureOpen();
        return readerToNormsProducer(reader);
      }

      @Override
      public DocValuesProducer getDocValuesReader() {
        reader.ensureOpen();
        return readerToDocValuesProducer(reader);
      }

      @Override
      public FieldsProducer getPostingsReader() {
        reader.ensureOpen();
        try {
          return readerToFieldsProducer(reader);
        } catch (IOException bogus) {
          throw new AssertionError(bogus);
        }
      }

      @Override
      public FieldInfos getFieldInfos() {
        return reader.getFieldInfos();
      }

      @Override
      public PointsReader getPointsReader() {
        return pointValuesToReader(reader);
      }

      @Override
      public Bits getLiveDocs() {
        return reader.getLiveDocs();
      }

      @Override
      public int numDocs() {
        return reader.numDocs();
      }

      @Override
      public int maxDoc() {
        return reader.maxDoc();
      }

      @Override
      public CacheHelper getCoreCacheHelper() {
        return reader.getCoreCacheHelper();
      }

      @Override
      public CacheHelper getReaderCacheHelper() {
        return reader.getReaderCacheHelper();
      }

      @Override
      public String toString() {
        return "SlowCodecReaderWrapper(" + reader + ")";
      }

      @Override
      public LeafMetaData getMetaData() {
        return reader.getMetaData();
      }
    };
  }
}
 
Example #15
Source File: MergeState.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Sole constructor. */
MergeState(List<CodecReader> originalReaders, SegmentInfo segmentInfo, InfoStream infoStream) throws IOException {

  this.infoStream = infoStream;

  final Sort indexSort = segmentInfo.getIndexSort();
  int numReaders = originalReaders.size();
  leafDocMaps = new DocMap[numReaders];
  List<CodecReader> readers = maybeSortReaders(originalReaders, segmentInfo);

  maxDocs = new int[numReaders];
  fieldsProducers = new FieldsProducer[numReaders];
  normsProducers = new NormsProducer[numReaders];
  storedFieldsReaders = new StoredFieldsReader[numReaders];
  termVectorsReaders = new TermVectorsReader[numReaders];
  docValuesProducers = new DocValuesProducer[numReaders];
  pointsReaders = new PointsReader[numReaders];
  fieldInfos = new FieldInfos[numReaders];
  liveDocs = new Bits[numReaders];

  int numDocs = 0;
  for(int i=0;i<numReaders;i++) {
    final CodecReader reader = readers.get(i);

    maxDocs[i] = reader.maxDoc();
    liveDocs[i] = reader.getLiveDocs();
    fieldInfos[i] = reader.getFieldInfos();

    normsProducers[i] = reader.getNormsReader();
    if (normsProducers[i] != null) {
      normsProducers[i] = normsProducers[i].getMergeInstance();
    }
    
    docValuesProducers[i] = reader.getDocValuesReader();
    if (docValuesProducers[i] != null) {
      docValuesProducers[i] = docValuesProducers[i].getMergeInstance();
    }
    
    storedFieldsReaders[i] = reader.getFieldsReader();
    if (storedFieldsReaders[i] != null) {
      storedFieldsReaders[i] = storedFieldsReaders[i].getMergeInstance();
    }
    
    termVectorsReaders[i] = reader.getTermVectorsReader();
    if (termVectorsReaders[i] != null) {
      termVectorsReaders[i] = termVectorsReaders[i].getMergeInstance();
    }
    
    fieldsProducers[i] = reader.getPostingsReader().getMergeInstance();
    pointsReaders[i] = reader.getPointsReader();
    if (pointsReaders[i] != null) {
      pointsReaders[i] = pointsReaders[i].getMergeInstance();
    }
    numDocs += reader.numDocs();
  }

  segmentInfo.setMaxDoc(numDocs);

  this.segmentInfo = segmentInfo;
  this.docMaps = buildDocMaps(readers, indexSort);
}
 
Example #16
Source File: SegmentReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader getFieldsReader() {
  ensureOpen();
  return core.fieldsReaderLocal.get();
}
 
Example #17
Source File: FilterCodecReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader getFieldsReader() {
  return in.getFieldsReader();
}
 
Example #18
Source File: CheckIndex.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/**
 * Test stored fields.
 * @lucene.experimental
 */
public static Status.StoredFieldStatus testStoredFields(CodecReader reader, PrintStream infoStream, boolean failFast) throws IOException {
  long startNS = System.nanoTime();
  final Status.StoredFieldStatus status = new Status.StoredFieldStatus();

  try {
    if (infoStream != null) {
      infoStream.print("    test: stored fields.......");
    }

    // Scan stored fields for all documents
    final Bits liveDocs = reader.getLiveDocs();
    StoredFieldsReader storedFields = reader.getFieldsReader().getMergeInstance();
    for (int j = 0; j < reader.maxDoc(); ++j) {
      // Intentionally pull even deleted documents to
      // make sure they too are not corrupt:
      DocumentStoredFieldVisitor visitor = new DocumentStoredFieldVisitor();
      storedFields.visitDocument(j, visitor);
      Document doc = visitor.getDocument();
      if (liveDocs == null || liveDocs.get(j)) {
        status.docCount++;
        status.totFields += doc.getFields().size();
      }
    }      

    // Validate docCount
    if (status.docCount != reader.numDocs()) {
      throw new RuntimeException("docCount=" + status.docCount + " but saw " + status.docCount + " undeleted docs");
    }

    msg(infoStream, String.format(Locale.ROOT, "OK [%d total field count; avg %.1f fields per doc] [took %.3f sec]",
                                  status.totFields,
                                  (((float) status.totFields)/status.docCount),
                                  nsToSec(System.nanoTime() - startNS)));
  } catch (Throwable e) {
    if (failFast) {
      throw IOUtils.rethrowAlways(e);
    }
    msg(infoStream, "ERROR [" + String.valueOf(e.getMessage()) + "]");
    status.error = e;
    if (infoStream != null) {
      e.printStackTrace(infoStream);
    }
  }

  return status;
}
 
Example #19
Source File: CompressingStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si,
    FieldInfos fn, IOContext context) throws IOException {
  return new CompressingStoredFieldsReader(directory, si, segmentSuffix, fn, 
      context, formatName, compressionMode);
}
 
Example #20
Source File: CompressingStoredFieldsReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader getMergeInstance() {
  ensureOpen();
  return new CompressingStoredFieldsReader(this, true);
}
 
Example #21
Source File: CompressingStoredFieldsReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader clone() {
  ensureOpen();
  return new CompressingStoredFieldsReader(this, false);
}
 
Example #22
Source File: MergingCodecReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader getFieldsReader() {
  return fieldsReader.get();
}
 
Example #23
Source File: MergingCodecReader.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
protected StoredFieldsReader initialValue() {
  return in.getFieldsReader().getMergeInstance();
}
 
Example #24
Source File: CrankyStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  return delegate.fieldsReader(directory, si, fn, context);
}
 
Example #25
Source File: AssertingStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader getMergeInstance() {
  return new AssertingStoredFieldsReader(in.getMergeInstance(), maxDoc, true);
}
 
Example #26
Source File: AssertingStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader clone() {
  assert merging == false : "Merge instances do not support cloning";
  return new AssertingStoredFieldsReader(in.clone(), maxDoc, false);
}
 
Example #27
Source File: AssertingStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {
  return new AssertingStoredFieldsReader(in.fieldsReader(directory, si, fn, context), si.maxDoc(), false);
}
 
Example #28
Source File: SimpleTextStoredFieldsFormat.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public StoredFieldsReader fieldsReader(Directory directory, SegmentInfo si, FieldInfos fn, IOContext context) throws IOException {;
  return new SimpleTextStoredFieldsReader(directory, si, fn, context);
}
 
Example #29
Source File: CodecReader.java    From lucene-solr with Apache License 2.0 2 votes vote down vote up
/** 
 * Expert: retrieve thread-private StoredFieldsReader
 * @lucene.internal 
 */
public abstract StoredFieldsReader getFieldsReader();