Java Code Examples for org.apache.lucene.index.IndexOptions#DOCS_AND_FREQS

The following examples show how to use org.apache.lucene.index.IndexOptions#DOCS_AND_FREQS . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: Lucene50FieldInfosFormat.java From lucene-solr with Apache License 2.0

6 votes

private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException {
  switch (b) {
  case 0:
    return IndexOptions.NONE;
  case 1:
    return IndexOptions.DOCS;
  case 2:
    return IndexOptions.DOCS_AND_FREQS;
  case 3:
    return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  case 4:
    return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  default:
    // BUG
    throw new CorruptIndexException("invalid IndexOptions byte: " + b, input);
  }
}

Example 2

Source File: Lucene60FieldInfosFormat.java From lucene-solr with Apache License 2.0

6 votes

private static IndexOptions getIndexOptions(IndexInput input, byte b) throws IOException {
  switch (b) {
  case 0:
    return IndexOptions.NONE;
  case 1:
    return IndexOptions.DOCS;
  case 2:
    return IndexOptions.DOCS_AND_FREQS;
  case 3:
    return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  case 4:
    return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  default:
    // BUG
    throw new CorruptIndexException("invalid IndexOptions byte: " + b, input);
  }
}

Example 3

Source File: TestBooleanSimilarity.java From lucene-solr with Apache License 2.0

6 votes

public void testSameNormsAsBM25() {
  BooleanSimilarity sim1 = new BooleanSimilarity();
  BM25Similarity sim2 = new BM25Similarity();
  sim2.setDiscountOverlaps(true);
  for (int iter = 0; iter < 100; ++iter) {
    final int length = TestUtil.nextInt(random(), 1, 100);
    final int position = random().nextInt(length);
    final int numOverlaps = random().nextInt(length);
    final int maxTermFrequency = 1;
    final int uniqueTermCount = 1;
    FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", IndexOptions.DOCS_AND_FREQS, position, length, numOverlaps, 100, maxTermFrequency, uniqueTermCount);
    assertEquals(
        sim2.computeNorm(state),
        sim1.computeNorm(state),
        0f);
  }
}

Example 4

Source File: TestClassicSimilarity.java From lucene-solr with Apache License 2.0

6 votes

public void testSameNormsAsBM25() {
  ClassicSimilarity sim1 = new ClassicSimilarity();
  BM25Similarity sim2 = new BM25Similarity();
  sim2.setDiscountOverlaps(true);
  for (int iter = 0; iter < 100; ++iter) {
    final int length = TestUtil.nextInt(random(), 1, 1000);
    final int position = random().nextInt(length);
    final int numOverlaps = random().nextInt(length);
    final int maxTermFrequency = 1;
    final int uniqueTermCount = 1;
    FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", IndexOptions.DOCS_AND_FREQS, position, length, numOverlaps, 100, maxTermFrequency, uniqueTermCount);
    assertEquals(
        sim2.computeNorm(state),
        sim1.computeNorm(state),
        0f);
  }
}

Example 5

Source File: SchemaField.java From lucene-solr with Apache License 2.0

6 votes

@Override
public IndexOptions indexOptions() {
  if (!indexed()) {
    return IndexOptions.NONE;
  }
  
  IndexOptions options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  if (omitTermFreqAndPositions()) {
    options = IndexOptions.DOCS;
  } else if (omitPositions()) {
    options = IndexOptions.DOCS_AND_FREQS;
  } else if (storeOffsetsWithPositions()) {
    options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  }

  return options;
}

Example 6

Source File: PreAnalyzedField.java From lucene-solr with Apache License 2.0

6 votes

/**
 * Utility method to create a {@link org.apache.lucene.document.FieldType}
 * based on the {@link SchemaField}
 */
public static org.apache.lucene.document.FieldType createFieldType(SchemaField field) {
  if (!field.indexed() && !field.stored()) {
    log.trace("Ignoring unindexed/unstored field: {}", field);
    return null;
  }
  org.apache.lucene.document.FieldType newType = new org.apache.lucene.document.FieldType();
  newType.setTokenized(field.isTokenized());
  newType.setStored(field.stored());
  newType.setOmitNorms(field.omitNorms());
  IndexOptions options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  if (field.omitTermFreqAndPositions()) {
    options = IndexOptions.DOCS;
  } else if (field.omitPositions()) {
    options = IndexOptions.DOCS_AND_FREQS;
  } else if (field.storeOffsetsWithPositions()) {
    options = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  }
  newType.setIndexOptions(options);
  newType.setStoreTermVectors(field.storeTermVector());
  newType.setStoreTermVectorOffsets(field.storeTermOffsets());
  newType.setStoreTermVectorPositions(field.storeTermPositions());
  newType.setStoreTermVectorPayloads(field.storeTermPayloads());
  return newType;
}

Example 7

Source File: TypeParsers.java From Elasticsearch with Apache License 2.0

5 votes

private static IndexOptions nodeIndexOptionValue(final Object propNode) {
    final String value = propNode.toString();
    if (INDEX_OPTIONS_OFFSETS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
    } else if (INDEX_OPTIONS_POSITIONS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
    } else if (INDEX_OPTIONS_FREQS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS;
    } else if (INDEX_OPTIONS_DOCS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS;
    } else {
        throw new ElasticsearchParseException("failed to parse index option [{}]", value);
    }
}

Example 8

Source File: TermVectorLeafReader.java From lucene-solr with Apache License 2.0

5 votes

public TermVectorLeafReader(String field, Terms terms) {
  fields = new Fields() {
    @Override
    public Iterator<String> iterator() {
      return Collections.singletonList(field).iterator();
    }

    @Override
    public Terms terms(String fld) throws IOException {
      if (!field.equals(fld)) {
        return null;
      }
      return terms;
    }

    @Override
    public int size() {
      return 1;
    }
  };

  IndexOptions indexOptions;
  if (!terms.hasFreqs()) {
    indexOptions = IndexOptions.DOCS;
  } else if (!terms.hasPositions()) {
    indexOptions = IndexOptions.DOCS_AND_FREQS;
  } else if (!terms.hasOffsets()) {
    indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
  } else {
    indexOptions = IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
  }
  FieldInfo fieldInfo = new FieldInfo(field, 0,
                                      true, true, terms.hasPayloads(),
                                      indexOptions, DocValuesType.NONE, -1, Collections.emptyMap(), 0, 0, 0, false);
  fieldInfos = new FieldInfos(new FieldInfo[]{fieldInfo});
}

Example 9

Source File: TestSimilarityBase.java From lucene-solr with Apache License 2.0

5 votes

public void testDiscountOverlapsBoost() throws IOException {
  BM25Similarity expected = new BM25Similarity();
  SimilarityBase actual = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
  expected.setDiscountOverlaps(false);
  actual.setDiscountOverlaps(false);
  FieldInvertState state = new FieldInvertState(Version.LATEST.major, "foo", IndexOptions.DOCS_AND_FREQS);
  state.setLength(5);
  state.setNumOverlap(2);
  assertEquals(expected.computeNorm(state), actual.computeNorm(state));
  expected.setDiscountOverlaps(true);
  actual.setDiscountOverlaps(true);
  assertEquals(expected.computeNorm(state), actual.computeNorm(state));
}

Example 10

Source File: TypeParsers.java From crate with Apache License 2.0

5 votes

private static IndexOptions nodeIndexOptionValue(final Object propNode) {
    final String value = propNode.toString();
    if (INDEX_OPTIONS_OFFSETS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS;
    } else if (INDEX_OPTIONS_POSITIONS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
    } else if (INDEX_OPTIONS_FREQS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS_AND_FREQS;
    } else if (INDEX_OPTIONS_DOCS.equalsIgnoreCase(value)) {
        return IndexOptions.DOCS;
    } else {
        throw new ElasticsearchParseException("failed to parse index option [{}]", value);
    }
}

Example 11

Source File: TextFieldMapper.java From crate with Apache License 2.0

4 votes

@Override
public TextFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
                + name + "] without positions enabled");
        }
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    setupFieldType(context);
    PrefixFieldMapper prefixMapper = null;
    if (prefixFieldType != null) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]");
        }
        // Copy the index options of the main field to allow phrase queries on
        // the prefix field.
        if (context.indexCreatedVersion().onOrAfter(Version.ES_V_6_5_1)) {
            if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
                // frequencies are not needed because prefix queries always use a constant score
                prefixFieldType.setIndexOptions(IndexOptions.DOCS);
            } else {
                prefixFieldType.setIndexOptions(fieldType.indexOptions());
            }
        } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
            prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
        }
        if (fieldType.storeTermVectorOffsets()) {
            prefixFieldType.setStoreTermVectorOffsets(true);
        }
        prefixFieldType.setAnalyzer(fieldType.indexAnalyzer());
        prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
    }
    if (fieldType().indexPhrases) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + name() + "]");
        }
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set index_phrases on field [" + name() + "] if positions are not enabled");
        }
    }
    return new TextFieldMapper(
        name,
        position,
        defaultExpression,
        fieldType(),
        defaultFieldType,
        positionIncrementGap,
        prefixMapper,
        context.indexSettings(),
        multiFieldsBuilder.build(this, context),
        copyTo
    );
}