Java Code Examples for org.apache.lucene.codecs.DocValuesFormat

The following examples show how to use org.apache.lucene.codecs.DocValuesFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: SegmentDocValues.java    License: Apache License 2.0 6 votes vote down vote up
private RefCount<DocValuesProducer> newDocValuesProducer(SegmentCommitInfo si, Directory dir, final Long gen, FieldInfos infos) throws IOException {
  Directory dvDir = dir;
  String segmentSuffix = "";
  if (gen.longValue() != -1) {
    dvDir = si.info.dir; // gen'd files are written outside CFS, so use SegInfo directory
    segmentSuffix = Long.toString(gen.longValue(), Character.MAX_RADIX);
  }

  // set SegmentReadState to list only the fields that are relevant to that gen
  SegmentReadState srs = new SegmentReadState(dvDir, si.info, infos, IOContext.READ, segmentSuffix);
  DocValuesFormat dvFormat = si.info.getCodec().docValuesFormat();
  return new RefCount<DocValuesProducer>(dvFormat.fieldsProducer(srs)) {
    @SuppressWarnings("synthetic-access")
    @Override
    protected void release() throws IOException {
      object.close();
      synchronized (SegmentDocValues.this) {
        genDVProducers.remove(gen);
      }
    }
  };
}
 
Example 2
Source Project: lucene-solr   Source File: SolrResourceLoader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reloads all Lucene SPI implementations using the new classloader.
 * This method must be called after {@link #addToClassLoader(List)}
 * and before using this ResourceLoader.
 */
synchronized void reloadLuceneSPI() {
  // TODO improve to use a static Set<URL> to check when we need to
  if (!needToReloadLuceneSPI) {
    return;
  }
  needToReloadLuceneSPI = false; // reset
  log.debug("Reloading Lucene SPI");

  // Codecs:
  PostingsFormat.reloadPostingsFormats(this.classLoader);
  DocValuesFormat.reloadDocValuesFormats(this.classLoader);
  Codec.reloadCodecs(this.classLoader);
  // Analysis:
  CharFilterFactory.reloadCharFilters(this.classLoader);
  TokenFilterFactory.reloadTokenFilters(this.classLoader);
  TokenizerFactory.reloadTokenizers(this.classLoader);
}
 
Example 3
Source Project: Elasticsearch   Source File: PluginsService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reloads all Lucene SPI implementations using the new classloader.
 * This method must be called after the new classloader has been created to
 * register the services for use.
 */
static void reloadLuceneSPI(ClassLoader loader) {
    // do NOT change the order of these method calls!

    // Codecs:
    PostingsFormat.reloadPostingsFormats(loader);
    DocValuesFormat.reloadDocValuesFormats(loader);
    Codec.reloadCodecs(loader);
    // Analysis:
    CharFilterFactory.reloadCharFilters(loader);
    TokenFilterFactory.reloadTokenFilters(loader);
    TokenizerFactory.reloadTokenizers(loader);
}
 
Example 4
Source Project: lucene-solr   Source File: CrankyDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
CrankyDocValuesFormat(DocValuesFormat delegate, Random random) {
  // we impersonate the passed-in codec, so we don't need to be in SPI,
  // and so we dont change file formats
  super(delegate.getName());
  this.delegate = delegate;
  this.random = random;
}
 
Example 5
Source Project: lucene-solr   Source File: RandomCodec.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String name) {
  DocValuesFormat codec = previousDVMappings.get(name);
  if (codec == null) {
    codec = dvFormats.get(Math.abs(perFieldSeed ^ name.hashCode()) % dvFormats.size());
    previousDVMappings.put(name, codec);
    // Safety:
    assert previousDVMappings.size() < 10000: "test went insane";
  }
  return codec;
}
 
Example 6
Source Project: lucene-solr   Source File: RandomCodec.java    License: Apache License 2.0 5 votes vote down vote up
private final void addDocValues(Set<String> avoidCodecs, DocValuesFormat... docvalues) {
  for (DocValuesFormat d : docvalues) {
    if (!avoidCodecs.contains(d.getName())) {
      dvFormats.add(d);
      dvFormatNames.add(d.getName());
    }
  }
}
 
Example 7
Source Project: lucene-solr   Source File: TestUtil.java    License: Apache License 2.0 5 votes vote down vote up
/** Return a Codec that can read any of the
 *  default codecs and formats, but always writes in the specified
 *  format. */
public static Codec alwaysDocValuesFormat(final DocValuesFormat format) {
  // TODO: we really need for docvalues impls etc to announce themselves
  // (and maybe their params, too) to infostream on flush and merge.
  // otherwise in a real debugging situation we won't know whats going on!
  if (LuceneTestCase.VERBOSE) {
    System.out.println("TestUtil: forcing docvalues format to:" + format);
  }
  return new AssertingCodec() {
    @Override
    public DocValuesFormat getDocValuesFormatForField(String field) {
      return format;
    }
  };
}
 
Example 8
Source Project: lucene-solr   Source File: TestUtil.java    License: Apache License 2.0 5 votes vote down vote up
public static String getDocValuesFormat(Codec codec, String field) {
  DocValuesFormat f = codec.docValuesFormat();
  if (f instanceof PerFieldDocValuesFormat) {
    return ((PerFieldDocValuesFormat) f).getDocValuesFormatForField(field).getName();
  } else {
    return f.getName();
  }
}
 
Example 9
Source Project: lucene-solr   Source File: PerFieldDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
public FieldsReader(final SegmentReadState readState) throws IOException {

      // Init each unique format:
      boolean success = false;
      try {
        // Read field name -> format name
        for (FieldInfo fi : readState.fieldInfos) {
          if (fi.getDocValuesType() != DocValuesType.NONE) {
            final String fieldName = fi.name;
            final String formatName = fi.getAttribute(PER_FIELD_FORMAT_KEY);
            if (formatName != null) {
              // null formatName means the field is in fieldInfos, but has no docvalues!
              final String suffix = fi.getAttribute(PER_FIELD_SUFFIX_KEY);
              if (suffix == null) {
                throw new IllegalStateException("missing attribute: " + PER_FIELD_SUFFIX_KEY + " for field: " + fieldName);
              }
              DocValuesFormat format = DocValuesFormat.forName(formatName);
              String segmentSuffix = getFullSegmentSuffix(readState.segmentSuffix, getSuffix(formatName, suffix));
              if (!formats.containsKey(segmentSuffix)) {
                formats.put(segmentSuffix, format.fieldsProducer(new SegmentReadState(readState, segmentSuffix)));
              }
              fields.put(fieldName, formats.get(segmentSuffix));
            }
          }
        }
        success = true;
      } finally {
        if (!success) {
          IOUtils.closeWhileHandlingException(formats.values());
        }
      }
    }
 
Example 10
Source Project: crate   Source File: PluginsService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Reloads all Lucene SPI implementations using the new classloader.
 * This method must be called after the new classloader has been created to
 * register the services for use.
 */
static void reloadLuceneSPI(ClassLoader loader) {
    // do NOT change the order of these method calls!

    // Codecs:
    PostingsFormat.reloadPostingsFormats(loader);
    DocValuesFormat.reloadDocValuesFormats(loader);
    Codec.reloadCodecs(loader);
    // Analysis:
    CharFilterFactory.reloadCharFilters(loader);
    TokenFilterFactory.reloadTokenFilters(loader);
    TokenizerFactory.reloadTokenizers(loader);
}
 
Example 11
Source Project: lucene-solr   Source File: SimpleTextCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat docValuesFormat() {
  return dvFormat;
}
 
Example 12
Source Project: lucene-solr   Source File: AssertingCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return AssertingCodec.this.getDocValuesFormatForField(field);
}
 
Example 13
Source Project: lucene-solr   Source File: AssertingCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat docValuesFormat() {
  return docValues;
}
 
Example 14
Source Project: lucene-solr   Source File: CrankyCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat docValuesFormat() {
  return new CrankyDocValuesFormat(delegate.docValuesFormat(), random);
}
 
Example 15
Source Project: lucene-solr   Source File: TestUtil.java    License: Apache License 2.0 4 votes vote down vote up
/** 
 * Returns the actual default docvalues format (e.g. LuceneMNDocValuesFormat for this version of Lucene.
 */
public static DocValuesFormat getDefaultDocValuesFormat() {
  return new Lucene80DocValuesFormat();
}
 
Example 16
Source Project: lucene-solr   Source File: Lucene86Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return Lucene86Codec.this.getDocValuesFormatForField(field);
}
 
Example 17
Source Project: lucene-solr   Source File: Lucene86Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}
 
Example 18
Source Project: lucene-solr   Source File: DefaultIndexingChain.java    License: Apache License 2.0 4 votes vote down vote up
/** Writes all buffered doc values (called from {@link #flush}). */
private void writeDocValues(SegmentWriteState state, Sorter.DocMap sortMap) throws IOException {
  int maxDoc = state.segmentInfo.maxDoc();
  DocValuesConsumer dvConsumer = null;
  boolean success = false;
  try {
    for (int i=0;i<fieldHash.length;i++) {
      PerField perField = fieldHash[i];
      while (perField != null) {
        if (perField.docValuesWriter != null) {
          if (perField.fieldInfo.getDocValuesType() == DocValuesType.NONE) {
            // BUG
            throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has no docValues but wrote them");
          }
          if (dvConsumer == null) {
            // lazy init
            DocValuesFormat fmt = state.segmentInfo.getCodec().docValuesFormat();
            dvConsumer = fmt.fieldsConsumer(state);
          }
          perField.docValuesWriter.flush(state, sortMap, dvConsumer);
          perField.docValuesWriter = null;
        } else if (perField.fieldInfo.getDocValuesType() != DocValuesType.NONE) {
          // BUG
          throw new AssertionError("segment=" + state.segmentInfo + ": field=\"" + perField.fieldInfo.name + "\" has docValues but did not write them");
        }
        perField = perField.next;
      }
    }

    // TODO: catch missing DV fields here?  else we have
    // null/"" depending on how docs landed in segments?
    // but we can't detect all cases, and we should leave
    // this behavior undefined. dv is not "schemaless": it's column-stride.
    success = true;
  } finally {
    if (success) {
      IOUtils.close(dvConsumer);
    } else {
      IOUtils.closeWhileHandlingException(dvConsumer);
    }
  }

  if (state.fieldInfos.hasDocValues() == false) {
    if (dvConsumer != null) {
      // BUG
      throw new AssertionError("segment=" + state.segmentInfo + ": fieldInfos has no docValues but wrote them");
    }
  } else if (dvConsumer == null) {
    // BUG
    throw new AssertionError("segment=" + state.segmentInfo + ": fieldInfos has docValues but did not wrote them");
  }
}
 
Example 19
Source Project: lucene-solr   Source File: TestPerFieldDocValuesFormat.java    License: Apache License 2.0 4 votes vote down vote up
MergeRecordingDocValueFormatWrapper(DocValuesFormat delegate) {
  super(delegate.getName());
  this.delegate = delegate;
}
 
Example 20
Source Project: lucene-solr   Source File: Lucene80Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return defaultDVFormat;
}
 
Example 21
Source Project: lucene-solr   Source File: Lucene80Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}
 
Example 22
Source Project: lucene-solr   Source File: Lucene84Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return Lucene84Codec.this.getDocValuesFormatForField(field);
}
 
Example 23
Source Project: lucene-solr   Source File: Lucene84Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}
 
Example 24
Source Project: mtas   Source File: MtasCodec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat docValuesFormat() {
  initDelegate();
  return delegate.docValuesFormat();
}
 
Example 25
Source Project: incubator-retired-blur   Source File: Blur021Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return Blur021Codec.this.getDocValuesFormatForField(field);
}
 
Example 26
Source Project: incubator-retired-blur   Source File: Blur021Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}
 
Example 27
Source Project: incubator-retired-blur   Source File: Blur022Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return Blur022Codec.this.getDocValuesFormatForField(field);
}
 
Example 28
Source Project: incubator-retired-blur   Source File: Blur022Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}
 
Example 29
Source Project: incubator-retired-blur   Source File: Blur024Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public DocValuesFormat getDocValuesFormatForField(String field) {
  return Blur024Codec.this.getDocValuesFormatForField(field);
}
 
Example 30
Source Project: incubator-retired-blur   Source File: Blur024Codec.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public final DocValuesFormat docValuesFormat() {
  return docValuesFormat;
}