org.elasticsearch.index.analysis.NamedAnalyzer Java Examples

The following examples show how to use org.elasticsearch.index.analysis.NamedAnalyzer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TokenCountFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
    TokenCountFieldMapper.Builder builder = tokenCountField(name);
    for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) {
        Map.Entry<String, Object> entry = iterator.next();
        String propName = Strings.toUnderscoreCase(entry.getKey());
        Object propNode = entry.getValue();
        if (propName.equals("null_value")) {
            builder.nullValue(nodeIntegerValue(propNode));
            iterator.remove();
        } else if (propName.equals("analyzer")) {
            NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString());
            if (analyzer == null) {
                throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]");
            }
            builder.analyzer(analyzer);
            iterator.remove();
        }
    }
    parseNumberField(builder, name, node, parserContext);
    if (builder.analyzer() == null) {
        throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't.");
    }
    return builder;
}
 
Example #2
Source File: SuggestUtils.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) {
    if (analyzer instanceof NamedAnalyzer) {
        analyzer = ((NamedAnalyzer)analyzer).analyzer();
    }
    if (analyzer instanceof CustomAnalyzer) {
        final CustomAnalyzer a = (CustomAnalyzer) analyzer;
        final TokenFilterFactory[] tokenFilters = a.tokenFilters();
        for (TokenFilterFactory tokenFilterFactory : tokenFilters) {
            if (tokenFilterFactory instanceof ShingleTokenFilterFactory) {
                return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory();
            } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) {
                return (ShingleTokenFilterFactory.Factory) tokenFilterFactory;
            }
        }
    }
    return null;
}
 
Example #3
Source File: IcuAnalysisTests.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 6 votes vote down vote up
public void testDefaultsIcuAnalysis() throws IOException {

        TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY,
                new BundlePlugin(Settings.EMPTY));

        CharFilterFactory charFilterFactory = analysis.charFilter.get("icu_normalizer");
        assertThat(charFilterFactory, instanceOf(IcuNormalizerCharFilterFactory.class));

        TokenizerFactory tf = analysis.tokenizer.get("icu_tokenizer");
        assertThat(tf, instanceOf(IcuTokenizerFactory.class));

        TokenFilterFactory filterFactory = analysis.tokenFilter.get("icu_normalizer");
        assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class));

        filterFactory = analysis.tokenFilter.get("icu_folding");
        assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class));

        filterFactory = analysis.tokenFilter.get("icu_transform");
        assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class));

        Analyzer analyzer = analysis.indexAnalyzers.get( "icu_collation");
        assertThat(analyzer, instanceOf(NamedAnalyzer.class));
    }
 
Example #4
Source File: MinHashFieldMapper.java    From elasticsearch-minhash with Apache License 2.0 6 votes vote down vote up
@Override
public Mapper.Builder parse(final String name, final Map<String, Object> node,
        final ParserContext parserContext) throws MapperParsingException {
    final MinHashFieldMapper.Builder builder = new MinHashFieldMapper.Builder(
            name);
    parseField(builder, name, node, parserContext);
    for (final Iterator<Map.Entry<String, Object>> iterator = node.entrySet()
            .iterator(); iterator.hasNext();) {
        final Map.Entry<String, Object> entry = iterator.next();
        final String propName = entry.getKey();
        final Object propNode = entry.getValue();
        if (propName.equals("minhash_analyzer") && propNode != null) {
            final NamedAnalyzer analyzer = parserContext
                    .getIndexAnalyzers().get(propNode.toString());
            builder.minhashAnalyzer(analyzer);
            iterator.remove();
        } else if (propName.equals("copy_bits_to")
                && propNode != null) {
            parseCopyBitsFields(propNode, builder);
            iterator.remove();
        }
    }
    return builder;
}
 
Example #5
Source File: KeywordFieldMapper.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public KeywordFieldMapper build(BuilderContext context) {
    setupFieldType(context);
    if (normalizerName != null) {
        NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName);
        if (normalizer == null) {
            throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]");
        }
        fieldType().setNormalizer(normalizer);
        final NamedAnalyzer searchAnalyzer;
        if (fieldType().splitQueriesOnWhitespace) {
            searchAnalyzer = indexAnalyzers.getWhitespaceNormalizer(normalizerName);
        } else {
            searchAnalyzer = normalizer;
        }
        fieldType().setSearchAnalyzer(searchAnalyzer);
    } else if (fieldType().splitQueriesOnWhitespace) {
        fieldType().setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer()));
    }
    return new KeywordFieldMapper(
        name,
        position,
        defaultExpression,
        fieldType,
        defaultFieldType,
        ignoreAbove,
        lengthLimit,
        context.indexSettings(),
        multiFieldsBuilder.build(this, context),
        copyTo
    );
}
 
Example #6
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings));
	mappers.mutator().putAll(fieldMappers).close();
	this.threadPool = threadPool;
}
 
Example #7
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = CopyOnWriteHashMap.copyOf(fieldMappers);
	this.threadPool = threadPool;
}
 
Example #8
Source File: OntologyMapper.java    From BioSolr with Apache License 2.0 5 votes vote down vote up
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues,
		NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer,
		PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat,
		SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings,
		Map<String, FieldMapper<String>> fieldMappers,
		ThreadPool threadPool) {
	super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null,
			fieldDataSettings, indexSettings, multiFields, null);
	this.ontologySettings = oSettings;
	// Mappers are added to mappers map as they are used/created
	this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings));
	mappers.mutator().putAll(fieldMappers).close();
	this.threadPool = threadPool;
}
 
Example #9
Source File: MinHashFieldMapper.java    From elasticsearch-minhash with Apache License 2.0 5 votes vote down vote up
protected MinHashFieldMapper(final String simpleName, final MappedFieldType fieldType,
        final MappedFieldType defaultFieldType, final Settings indexSettings,
        final MultiFields multiFields, final CopyTo copyTo,
        final NamedAnalyzer minhashAnalyzer, final CopyBitsTo copyBitsTo) {
    super(simpleName, fieldType, defaultFieldType, indexSettings,
            multiFields, copyTo);
    this.minhashAnalyzer = minhashAnalyzer;
    this.copyBitsTo = copyBitsTo;
}
 
Example #10
Source File: StringFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public StringFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    // if the field is not analyzed, then by default, we should omit norms and have docs only
    // index options, as probably what the user really wants
    // if they are set explicitly, we will use those values
    // we also change the values on the default field type so that toXContent emits what
    // differs from the defaults
    if (fieldType.indexOptions() != IndexOptions.NONE && !fieldType.tokenized()) {
        defaultFieldType.setOmitNorms(true);
        defaultFieldType.setIndexOptions(IndexOptions.DOCS);
        if (!omitNormsSet && fieldType.boost() == 1.0f) {
            fieldType.setOmitNorms(true);
        }
        if (!indexOptionsSet) {
            fieldType.setIndexOptions(IndexOptions.DOCS);
        }
    }
    setupFieldType(context);
    StringFieldMapper fieldMapper = new StringFieldMapper(
            name, fieldType, defaultFieldType, positionIncrementGap, ignoreAbove,
            context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo);
    return fieldMapper.includeInAll(includeInAll);
}
 
Example #11
Source File: CompletionFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) {
    NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name);
    if (analyzer == null) {
        throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]");
    }
    return analyzer;
}
 
Example #12
Source File: TranslogHandler.java    From crate with Apache License 2.0 5 votes vote down vote up
public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings indexSettings) {
    NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer());
    IndexAnalyzers indexAnalyzers =
            new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, emptyMap(), emptyMap(), emptyMap());
    MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry();
    mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, mapperRegistry,
            () -> null);
}
 
Example #13
Source File: ArrayFieldType.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public NamedAnalyzer indexAnalyzer() {
    return innerFieldType.indexAnalyzer();
}
 
Example #14
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void normalizeOther(MappedFieldType other) {
    other.setSearchAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #15
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void modify(MappedFieldType ft) {
    ft.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #16
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void normalizeOther(MappedFieldType other) {
    other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #17
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void modify(MappedFieldType ft) {
    ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #18
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void modify(MappedFieldType ft) {
    ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #19
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void modify(MappedFieldType ft) {
    ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #20
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public NamedAnalyzer searchQuoteAnalyzer() {
    return innerFieldType.searchQuoteAnalyzer();
}
 
Example #21
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void normalizeOther(MappedFieldType other) {
    other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}
 
Example #22
Source File: TextFieldMapper.java    From crate with Apache License 2.0 4 votes vote down vote up
void setAnalyzer(String name, Analyzer delegate) {
    setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate)));
}
 
Example #23
Source File: TextFieldMapper.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public TextFieldMapper build(BuilderContext context) {
    if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) {
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set position_increment_gap on field ["
                + name + "] without positions enabled");
        }
        fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap));
        fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap));
        fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap));
    }
    setupFieldType(context);
    PrefixFieldMapper prefixMapper = null;
    if (prefixFieldType != null) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]");
        }
        // Copy the index options of the main field to allow phrase queries on
        // the prefix field.
        if (context.indexCreatedVersion().onOrAfter(Version.ES_V_6_5_1)) {
            if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
                // frequencies are not needed because prefix queries always use a constant score
                prefixFieldType.setIndexOptions(IndexOptions.DOCS);
            } else {
                prefixFieldType.setIndexOptions(fieldType.indexOptions());
            }
        } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) {
            prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS);
        }
        if (fieldType.storeTermVectorOffsets()) {
            prefixFieldType.setStoreTermVectorOffsets(true);
        }
        prefixFieldType.setAnalyzer(fieldType.indexAnalyzer());
        prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
    }
    if (fieldType().indexPhrases) {
        if (fieldType().isSearchable() == false) {
            throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + name() + "]");
        }
        if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
            throw new IllegalArgumentException("Cannot set index_phrases on field [" + name() + "] if positions are not enabled");
        }
    }
    return new TextFieldMapper(
        name,
        position,
        defaultExpression,
        fieldType(),
        defaultFieldType,
        positionIncrementGap,
        prefixMapper,
        context.indexSettings(),
        multiFieldsBuilder.build(this, context),
        copyTo
    );
}
 
Example #24
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void setSearchQuoteAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setSearchQuoteAnalyzer(analyzer);
}
 
Example #25
Source File: KeywordFieldMapper.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException {
    String value;
    if (context.externalValueSet()) {
        value = context.externalValue().toString();
    } else {
        XContentParser parser = context.parser();
        if (parser.currentToken() == XContentParser.Token.VALUE_NULL) {
            value = fieldType().nullValueAsString();
        } else {
            value = parser.textOrNull();
        }
    }

    if (value == null || value.length() > ignoreAbove) {
        return;
    }

    final NamedAnalyzer normalizer = fieldType().normalizer();
    if (normalizer != null) {
        try (TokenStream ts = normalizer.tokenStream(name(), value)) {
            final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
            ts.reset();
            if (ts.incrementToken() == false) {
                throw new IllegalStateException("The normalization token stream is "
                    + "expected to produce exactly 1 token, but got 0 for analyzer " + normalizer
                    + " and input \"" + value + "\"");
            }
            final String newValue = termAtt.toString();
            if (ts.incrementToken()) {
                throw new IllegalStateException("The normalization token stream is "
                    + "expected to produce exactly 1 token, but got 2+ for analyzer " + normalizer
                    + " and input \"" + value + "\"");
            }
            ts.end();
            value = newValue;
        }
    }

    // convert to utf8 only once before feeding postings/dv/stored fields
    final BytesRef binaryValue = new BytesRef(value);
    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().name(), binaryValue, fieldType());
        fields.add(field);

        if (fieldType().hasDocValues() == false && fieldType().omitNorms()) {
            createFieldNamesField(context, fields);
        }
    }

    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().name(), binaryValue));
    }
}
 
Example #26
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void setSearchAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setSearchAnalyzer(analyzer);
}
 
Example #27
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public NamedAnalyzer searchAnalyzer() {
    return innerFieldType.searchAnalyzer();
}
 
Example #28
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void setIndexAnalyzer(NamedAnalyzer analyzer) {
    innerFieldType.setIndexAnalyzer(analyzer);
}
 
Example #29
Source File: ArrayFieldType.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public NamedAnalyzer indexAnalyzer() {
    return innerFieldType.indexAnalyzer();
}
 
Example #30
Source File: FieldTypeTestCase.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public void normalizeOther(MappedFieldType other) {
    other.setSearchQuoteAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer()));
}