org.elasticsearch.index.analysis.NamedAnalyzer Java Examples
The following examples show how to use
org.elasticsearch.index.analysis.NamedAnalyzer.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TokenCountFieldMapper.java From Elasticsearch with Apache License 2.0 | 6 votes |
@Override @SuppressWarnings("unchecked") public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException { TokenCountFieldMapper.Builder builder = tokenCountField(name); for (Iterator<Map.Entry<String, Object>> iterator = node.entrySet().iterator(); iterator.hasNext();) { Map.Entry<String, Object> entry = iterator.next(); String propName = Strings.toUnderscoreCase(entry.getKey()); Object propNode = entry.getValue(); if (propName.equals("null_value")) { builder.nullValue(nodeIntegerValue(propNode)); iterator.remove(); } else if (propName.equals("analyzer")) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(propNode.toString()); if (analyzer == null) { throw new MapperParsingException("Analyzer [" + propNode.toString() + "] not found for field [" + name + "]"); } builder.analyzer(analyzer); iterator.remove(); } } parseNumberField(builder, name, node, parserContext); if (builder.analyzer() == null) { throw new MapperParsingException("Analyzer must be set for field [" + name + "] but wasn't."); } return builder; }
Example #2
Source File: SuggestUtils.java From Elasticsearch with Apache License 2.0 | 6 votes |
public static ShingleTokenFilterFactory.Factory getShingleFilterFactory(Analyzer analyzer) { if (analyzer instanceof NamedAnalyzer) { analyzer = ((NamedAnalyzer)analyzer).analyzer(); } if (analyzer instanceof CustomAnalyzer) { final CustomAnalyzer a = (CustomAnalyzer) analyzer; final TokenFilterFactory[] tokenFilters = a.tokenFilters(); for (TokenFilterFactory tokenFilterFactory : tokenFilters) { if (tokenFilterFactory instanceof ShingleTokenFilterFactory) { return ((ShingleTokenFilterFactory)tokenFilterFactory).getInnerFactory(); } else if (tokenFilterFactory instanceof ShingleTokenFilterFactory.Factory) { return (ShingleTokenFilterFactory.Factory) tokenFilterFactory; } } } return null; }
Example #3
Source File: IcuAnalysisTests.java From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 | 6 votes |
public void testDefaultsIcuAnalysis() throws IOException { TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new BundlePlugin(Settings.EMPTY)); CharFilterFactory charFilterFactory = analysis.charFilter.get("icu_normalizer"); assertThat(charFilterFactory, instanceOf(IcuNormalizerCharFilterFactory.class)); TokenizerFactory tf = analysis.tokenizer.get("icu_tokenizer"); assertThat(tf, instanceOf(IcuTokenizerFactory.class)); TokenFilterFactory filterFactory = analysis.tokenFilter.get("icu_normalizer"); assertThat(filterFactory, instanceOf(IcuNormalizerTokenFilterFactory.class)); filterFactory = analysis.tokenFilter.get("icu_folding"); assertThat(filterFactory, instanceOf(IcuFoldingTokenFilterFactory.class)); filterFactory = analysis.tokenFilter.get("icu_transform"); assertThat(filterFactory, instanceOf(IcuTransformTokenFilterFactory.class)); Analyzer analyzer = analysis.indexAnalyzers.get( "icu_collation"); assertThat(analyzer, instanceOf(NamedAnalyzer.class)); }
Example #4
Source File: MinHashFieldMapper.java From elasticsearch-minhash with Apache License 2.0 | 6 votes |
@Override public Mapper.Builder parse(final String name, final Map<String, Object> node, final ParserContext parserContext) throws MapperParsingException { final MinHashFieldMapper.Builder builder = new MinHashFieldMapper.Builder( name); parseField(builder, name, node, parserContext); for (final Iterator<Map.Entry<String, Object>> iterator = node.entrySet() .iterator(); iterator.hasNext();) { final Map.Entry<String, Object> entry = iterator.next(); final String propName = entry.getKey(); final Object propNode = entry.getValue(); if (propName.equals("minhash_analyzer") && propNode != null) { final NamedAnalyzer analyzer = parserContext .getIndexAnalyzers().get(propNode.toString()); builder.minhashAnalyzer(analyzer); iterator.remove(); } else if (propName.equals("copy_bits_to") && propNode != null) { parseCopyBitsFields(propNode, builder); iterator.remove(); } } return builder; }
Example #5
Source File: KeywordFieldMapper.java From crate with Apache License 2.0 | 5 votes |
@Override public KeywordFieldMapper build(BuilderContext context) { setupFieldType(context); if (normalizerName != null) { NamedAnalyzer normalizer = indexAnalyzers.getNormalizer(normalizerName); if (normalizer == null) { throw new MapperParsingException("normalizer [" + normalizerName + "] not found for field [" + name + "]"); } fieldType().setNormalizer(normalizer); final NamedAnalyzer searchAnalyzer; if (fieldType().splitQueriesOnWhitespace) { searchAnalyzer = indexAnalyzers.getWhitespaceNormalizer(normalizerName); } else { searchAnalyzer = normalizer; } fieldType().setSearchAnalyzer(searchAnalyzer); } else if (fieldType().splitQueriesOnWhitespace) { fieldType().setSearchAnalyzer(new NamedAnalyzer("whitespace", AnalyzerScope.INDEX, new WhitespaceAnalyzer())); } return new KeywordFieldMapper( name, position, defaultExpression, fieldType, defaultFieldType, ignoreAbove, lengthLimit, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo ); }
Example #6
Source File: OntologyMapper.java From BioSolr with Apache License 2.0 | 5 votes |
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat, SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings, Map<String, FieldMapper<String>> fieldMappers, ThreadPool threadPool) { super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null, fieldDataSettings, indexSettings, multiFields, null); this.ontologySettings = oSettings; // Mappers are added to mappers map as they are used/created this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings)); mappers.mutator().putAll(fieldMappers).close(); this.threadPool = threadPool; }
Example #7
Source File: OntologyMapper.java From BioSolr with Apache License 2.0 | 5 votes |
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat, SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings, Map<String, FieldMapper<String>> fieldMappers, ThreadPool threadPool) { super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null, fieldDataSettings, indexSettings, multiFields, null); this.ontologySettings = oSettings; // Mappers are added to mappers map as they are used/created this.mappers = CopyOnWriteHashMap.copyOf(fieldMappers); this.threadPool = threadPool; }
Example #8
Source File: OntologyMapper.java From BioSolr with Apache License 2.0 | 5 votes |
public OntologyMapper(FieldMapper.Names names, FieldType fieldType, Boolean docValues, NamedAnalyzer indexAnalyzer, NamedAnalyzer searchAnalyzer, PostingsFormatProvider postingsFormat, DocValuesFormatProvider docValuesFormat, SimilarityProvider similarity, @Nullable Settings fieldDataSettings, Settings indexSettings, MultiFields multiFields, OntologySettings oSettings, Map<String, FieldMapper<String>> fieldMappers, ThreadPool threadPool) { super(names, 1f, fieldType, docValues, searchAnalyzer, indexAnalyzer, postingsFormat, docValuesFormat, similarity, null, fieldDataSettings, indexSettings, multiFields, null); this.ontologySettings = oSettings; // Mappers are added to mappers map as they are used/created this.mappers = UpdateInPlaceMap.of(MapperService.getFieldMappersCollectionSwitch(indexSettings)); mappers.mutator().putAll(fieldMappers).close(); this.threadPool = threadPool; }
Example #9
Source File: MinHashFieldMapper.java From elasticsearch-minhash with Apache License 2.0 | 5 votes |
protected MinHashFieldMapper(final String simpleName, final MappedFieldType fieldType, final MappedFieldType defaultFieldType, final Settings indexSettings, final MultiFields multiFields, final CopyTo copyTo, final NamedAnalyzer minhashAnalyzer, final CopyBitsTo copyBitsTo) { super(simpleName, fieldType, defaultFieldType, indexSettings, multiFields, copyTo); this.minhashAnalyzer = minhashAnalyzer; this.copyBitsTo = copyBitsTo; }
Example #10
Source File: StringFieldMapper.java From Elasticsearch with Apache License 2.0 | 5 votes |
@Override public StringFieldMapper build(BuilderContext context) { if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) { fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap)); fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap)); fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap)); } // if the field is not analyzed, then by default, we should omit norms and have docs only // index options, as probably what the user really wants // if they are set explicitly, we will use those values // we also change the values on the default field type so that toXContent emits what // differs from the defaults if (fieldType.indexOptions() != IndexOptions.NONE && !fieldType.tokenized()) { defaultFieldType.setOmitNorms(true); defaultFieldType.setIndexOptions(IndexOptions.DOCS); if (!omitNormsSet && fieldType.boost() == 1.0f) { fieldType.setOmitNorms(true); } if (!indexOptionsSet) { fieldType.setIndexOptions(IndexOptions.DOCS); } } setupFieldType(context); StringFieldMapper fieldMapper = new StringFieldMapper( name, fieldType, defaultFieldType, positionIncrementGap, ignoreAbove, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); return fieldMapper.includeInAll(includeInAll); }
Example #11
Source File: CompletionFieldMapper.java From Elasticsearch with Apache License 2.0 | 5 votes |
private NamedAnalyzer getNamedAnalyzer(ParserContext parserContext, String name) { NamedAnalyzer analyzer = parserContext.analysisService().analyzer(name); if (analyzer == null) { throw new IllegalArgumentException("Can't find default or mapped analyzer with name [" + name + "]"); } return analyzer; }
Example #12
Source File: TranslogHandler.java From crate with Apache License 2.0 | 5 votes |
public TranslogHandler(NamedXContentRegistry xContentRegistry, IndexSettings indexSettings) { NamedAnalyzer defaultAnalyzer = new NamedAnalyzer("default", AnalyzerScope.INDEX, new StandardAnalyzer()); IndexAnalyzers indexAnalyzers = new IndexAnalyzers(indexSettings, defaultAnalyzer, defaultAnalyzer, defaultAnalyzer, emptyMap(), emptyMap(), emptyMap()); MapperRegistry mapperRegistry = new IndicesModule(emptyList()).getMapperRegistry(); mapperService = new MapperService(indexSettings, indexAnalyzers, xContentRegistry, mapperRegistry, () -> null); }
Example #13
Source File: ArrayFieldType.java From Elasticsearch with Apache License 2.0 | 4 votes |
@Override public NamedAnalyzer indexAnalyzer() { return innerFieldType.indexAnalyzer(); }
Example #14
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void normalizeOther(MappedFieldType other) { other.setSearchAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #15
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void modify(MappedFieldType ft) { ft.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #16
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void normalizeOther(MappedFieldType other) { other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #17
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void modify(MappedFieldType ft) { ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #18
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void modify(MappedFieldType ft) { ft.setSearchAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #19
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void modify(MappedFieldType ft) { ft.setSearchQuoteAnalyzer(new NamedAnalyzer("bar", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #20
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public NamedAnalyzer searchQuoteAnalyzer() { return innerFieldType.searchQuoteAnalyzer(); }
Example #21
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void normalizeOther(MappedFieldType other) { other.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer())); }
Example #22
Source File: TextFieldMapper.java From crate with Apache License 2.0 | 4 votes |
void setAnalyzer(String name, Analyzer delegate) { setIndexAnalyzer(new NamedAnalyzer(name, AnalyzerScope.INDEX, new PhraseWrappedAnalyzer(delegate))); }
Example #23
Source File: TextFieldMapper.java From crate with Apache License 2.0 | 4 votes |
@Override public TextFieldMapper build(BuilderContext context) { if (positionIncrementGap != POSITION_INCREMENT_GAP_USE_ANALYZER) { if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException("Cannot set position_increment_gap on field [" + name + "] without positions enabled"); } fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap)); fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap)); fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap)); } setupFieldType(context); PrefixFieldMapper prefixMapper = null; if (prefixFieldType != null) { if (fieldType().isSearchable() == false) { throw new IllegalArgumentException("Cannot set index_prefixes on unindexed field [" + name() + "]"); } // Copy the index options of the main field to allow phrase queries on // the prefix field. if (context.indexCreatedVersion().onOrAfter(Version.ES_V_6_5_1)) { if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) { // frequencies are not needed because prefix queries always use a constant score prefixFieldType.setIndexOptions(IndexOptions.DOCS); } else { prefixFieldType.setIndexOptions(fieldType.indexOptions()); } } else if (fieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) { prefixFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS); } if (fieldType.storeTermVectorOffsets()) { prefixFieldType.setStoreTermVectorOffsets(true); } prefixFieldType.setAnalyzer(fieldType.indexAnalyzer()); prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings()); } if (fieldType().indexPhrases) { if (fieldType().isSearchable() == false) { throw new IllegalArgumentException("Cannot set index_phrases on unindexed field [" + name() + "]"); } if (fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { throw new IllegalArgumentException("Cannot set index_phrases on field [" + name() + "] if positions are not enabled"); } } return new TextFieldMapper( name, position, defaultExpression, fieldType(), defaultFieldType, positionIncrementGap, prefixMapper, context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo ); }
Example #24
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public void setSearchQuoteAnalyzer(NamedAnalyzer analyzer) { innerFieldType.setSearchQuoteAnalyzer(analyzer); }
Example #25
Source File: KeywordFieldMapper.java From crate with Apache License 2.0 | 4 votes |
@Override protected void parseCreateField(ParseContext context, List<IndexableField> fields) throws IOException { String value; if (context.externalValueSet()) { value = context.externalValue().toString(); } else { XContentParser parser = context.parser(); if (parser.currentToken() == XContentParser.Token.VALUE_NULL) { value = fieldType().nullValueAsString(); } else { value = parser.textOrNull(); } } if (value == null || value.length() > ignoreAbove) { return; } final NamedAnalyzer normalizer = fieldType().normalizer(); if (normalizer != null) { try (TokenStream ts = normalizer.tokenStream(name(), value)) { final CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); if (ts.incrementToken() == false) { throw new IllegalStateException("The normalization token stream is " + "expected to produce exactly 1 token, but got 0 for analyzer " + normalizer + " and input \"" + value + "\""); } final String newValue = termAtt.toString(); if (ts.incrementToken()) { throw new IllegalStateException("The normalization token stream is " + "expected to produce exactly 1 token, but got 2+ for analyzer " + normalizer + " and input \"" + value + "\""); } ts.end(); value = newValue; } } // convert to utf8 only once before feeding postings/dv/stored fields final BytesRef binaryValue = new BytesRef(value); if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { Field field = new Field(fieldType().name(), binaryValue, fieldType()); fields.add(field); if (fieldType().hasDocValues() == false && fieldType().omitNorms()) { createFieldNamesField(context, fields); } } if (fieldType().hasDocValues()) { fields.add(new SortedSetDocValuesField(fieldType().name(), binaryValue)); } }
Example #26
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public void setSearchAnalyzer(NamedAnalyzer analyzer) { innerFieldType.setSearchAnalyzer(analyzer); }
Example #27
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public NamedAnalyzer searchAnalyzer() { return innerFieldType.searchAnalyzer(); }
Example #28
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public void setIndexAnalyzer(NamedAnalyzer analyzer) { innerFieldType.setIndexAnalyzer(analyzer); }
Example #29
Source File: ArrayFieldType.java From crate with Apache License 2.0 | 4 votes |
@Override public NamedAnalyzer indexAnalyzer() { return innerFieldType.indexAnalyzer(); }
Example #30
Source File: FieldTypeTestCase.java From crate with Apache License 2.0 | 4 votes |
@Override public void normalizeOther(MappedFieldType other) { other.setSearchQuoteAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.INDEX, new StandardAnalyzer())); }