Java Code Examples for org.elasticsearch.index.mapper.MapperService

The following examples show how to use org.elasticsearch.index.mapper.MapperService. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Override
protected FieldStatsShardResponse shardOperation(FieldStatsShardRequest request) {
    ShardId shardId = request.shardId();
    Map<String, FieldStats> fieldStats = new HashMap<>();
    IndexService indexServices = indicesService.indexServiceSafe(shardId.getIndex());
    MapperService mapperService = indexServices.mapperService();
    IndexShard shard = indexServices.shardSafe(shardId.id());
    try (Engine.Searcher searcher = shard.acquireSearcher("fieldstats")) {
        for (String field : request.getFields()) {
            MappedFieldType fieldType = mapperService.fullName(field);
            if (fieldType != null) {
                IndexReader reader = searcher.reader();
                Terms terms = MultiFields.getTerms(reader, field);
                if (terms != null) {
                    fieldStats.put(field, fieldType.stats(terms, reader.maxDoc()));
                }
            } else {
                throw new IllegalArgumentException("field [" + field + "] doesn't exist");
            }
        }
    } catch (IOException e) {
        throw ExceptionsHelper.convertToElastic(e);
    }
    return new FieldStatsShardResponse(shardId, fieldStats);
}
 
Example 2
Source Project: crate   Source File: ArrayMapperTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * create index with type and mapping and validate DocumentMapper serialization
 */
private DocumentMapper mapper(String indexName, String mapping) throws IOException {
    IndicesModule indicesModule = new IndicesModule(Collections.singletonList(new MapperPlugin() {
        @Override
        public Map<String, Mapper.TypeParser> getMappers() {
            return Collections.singletonMap(ArrayMapper.CONTENT_TYPE, new ArrayTypeParser());
        }
    }));
    MapperService mapperService = MapperTestUtils.newMapperService(
        NamedXContentRegistry.EMPTY,
        createTempDir(),
        Settings.EMPTY,
        indicesModule,
        indexName
    );
    DocumentMapperParser parser = mapperService.documentMapperParser();

    DocumentMapper defaultMapper = parser.parse(TYPE, new CompressedXContent(mapping));
    XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
    builder.startObject();
    defaultMapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
    builder.endObject();
    String rebuildMapping = Strings.toString(builder);
    return parser.parse(TYPE, new CompressedXContent(rebuildMapping));
}
 
Example 3
Source Project: elasticshell   Source File: ClientScopeSynchronizer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Retrieves the current indexes and types from elasticsearch
 * @return a set containing the indexes available in the elasticsearch cluster and their types
 */
protected Set<Index> getIndexes() {
    ClusterStateResponse response = unwrapShellNativeClient().client().admin().cluster().prepareState().setFilterBlocks(true)
            .setFilterRoutingTable(true).setFilterNodes(true).execute().actionGet();

    Set<Index> newIndexes = new HashSet<Index>();
    for (IndexMetaData indexMetaData : response.getState().metaData().indices().values()) {
        logger.trace("Processing index {}", indexMetaData.index());

        Set<String> typeNames = Sets.filter(indexMetaData.mappings().keySet(), new Predicate<String>() {
            @Override
            public boolean apply(String s) {
                return !MapperService.DEFAULT_MAPPING.equals(s);
            }
        });
        String[] types = typeNames.toArray(new String[typeNames.size()]);

        newIndexes.add(new Index(indexMetaData.index(), false, types));

        for (String alias : indexMetaData.aliases().keySet()) {
            newIndexes.add(new Index(alias, true, types));
        }
    }
    return newIndexes;
}
 
Example 4
Source Project: Elasticsearch   Source File: FetchCollector.java    License: Apache License 2.0 6 votes vote down vote up
public FetchCollector(List<LuceneCollectorExpression<?>> collectorExpressions,
                      MapperService mapperService,
                      Engine.Searcher searcher,
                      IndexFieldDataService indexFieldDataService,
                      int readerId) {
    this.collectorExpressions = collectorExpressions;
    this.readerContexts = searcher.searcher().getIndexReader().leaves();
    this.fieldsVisitor = new CollectorFieldsVisitor(this.collectorExpressions.size());
    CollectorContext collectorContext = new CollectorContext(mapperService, indexFieldDataService, fieldsVisitor, readerId);
    for (LuceneCollectorExpression<?> collectorExpression : this.collectorExpressions) {
        collectorExpression.startCollect(collectorContext);
    }
    visitorEnabled = fieldsVisitor.required();
    this.row = new InputRow(collectorExpressions);

}
 
Example 5
Source Project: Elasticsearch   Source File: MatchQueryBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public MatchQueryBuilder(MapperService mapperService,
                         IndexCache indexCache,
                         @Nullable BytesRef matchType,
                         @Nullable Map options) throws IOException {
    this.mapperService = mapperService;
    this.indexCache = indexCache;
    if (matchType == null) {
        this.matchType = MultiMatchQueryBuilder.Type.BEST_FIELDS;
    } else {
        this.matchType = SUPPORTED_TYPES.get(matchType);
        if (this.matchType == null) {
            throw illegalMatchType(BytesRefs.toString(matchType));
        }
    }
    this.options = OptionParser.parse(this.matchType, options);
}
 
Example 6
Source Project: Elasticsearch   Source File: LuceneQueryBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public Context convert(WhereClause whereClause,
                       MapperService mapperService,
                       IndexFieldDataService indexFieldDataService,
                       IndexCache indexCache) throws UnsupportedFeatureException {
    Context ctx = new Context(inputSymbolVisitor, mapperService, indexFieldDataService, indexCache);
    if (whereClause.noMatch()) {
        ctx.query = Queries.newMatchNoDocsQuery();
    } else if (!whereClause.hasQuery()) {
        ctx.query = Queries.newMatchAllQuery();
    } else {
        ctx.query = VISITOR.process(whereClause.query(), ctx);
    }
    if (LOGGER.isTraceEnabled()) {
        if (whereClause.hasQuery()) {
            LOGGER.trace("WHERE CLAUSE [{}] -> LUCENE QUERY [{}] ", SymbolPrinter.INSTANCE.printSimple(whereClause.query()), ctx.query);
        }
    }
    return ctx;
}
 
Example 7
Source Project: Elasticsearch   Source File: AutoCreateIndex.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public AutoCreateIndex(Settings settings, IndexNameExpressionResolver resolver) {
    this.resolver = resolver;
    dynamicMappingDisabled = !settings.getAsBoolean(MapperService.INDEX_MAPPER_DYNAMIC_SETTING, MapperService.INDEX_MAPPER_DYNAMIC_DEFAULT);
    String value = settings.get("action.auto_create_index");
    if (value == null || Booleans.isExplicitTrue(value)) {
        needToCheck = true;
        globallyDisabled = false;
        matches = null;
        matches2 = null;
    } else if (Booleans.isExplicitFalse(value)) {
        needToCheck = false;
        globallyDisabled = true;
        matches = null;
        matches2 = null;
    } else {
        needToCheck = true;
        globallyDisabled = false;
        matches = Strings.commaDelimitedListToStringArray(value);
        matches2 = new String[matches.length];
        for (int i = 0; i < matches.length; i++) {
            matches2[i] = matches[i].substring(1);
        }
    }
}
 
Example 8
Source Project: crate   Source File: InternalEngine.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean hasCompleteOperationHistory(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        return getMinRetainedSeqNo() <= startingSeqNo;
    } else {
        final long currentLocalCheckpoint = getLocalCheckpointTracker().getProcessedCheckpoint();
        final LocalCheckpointTracker tracker = new LocalCheckpointTracker(startingSeqNo, startingSeqNo - 1);
        try (Translog.Snapshot snapshot = getTranslog().newSnapshotFromMinSeqNo(startingSeqNo)) {
            Translog.Operation operation;
            while ((operation = snapshot.next()) != null) {
                if (operation.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO) {
                    tracker.markSeqNoAsProcessed(operation.seqNo());
                }
            }
        }
        return tracker.getProcessedCheckpoint() >= currentLocalCheckpoint;
    }
}
 
Example 9
Source Project: Elasticsearch   Source File: PercolatorService.java    License: Apache License 2.0 6 votes vote down vote up
private ParsedDocument parseFetchedDoc(PercolateContext context, BytesReference fetchedDoc, IndexService documentIndexService, String index, String type) {
    ParsedDocument doc = null;
    XContentParser parser = null;
    try {
        parser = XContentFactory.xContent(fetchedDoc).createParser(fetchedDoc);
        MapperService mapperService = documentIndexService.mapperService();
        DocumentMapperForType docMapper = mapperService.documentMapperWithAutoCreate(type);
        doc = docMapper.getDocumentMapper().parse(source(parser).index(index).type(type).flyweight(true));

        if (context.highlight() != null) {
            doc.setSource(fetchedDoc);
        }
    } catch (Throwable e) {
        throw new ElasticsearchParseException("failed to parse request", e);
    } finally {
        if (parser != null) {
            parser.close();
        }
    }

    if (doc == null) {
        throw new ElasticsearchParseException("No doc to percolate in the request");
    }

    return doc;
}
 
Example 10
Source Project: Elasticsearch   Source File: SuggestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static boolean parseSuggestContext(XContentParser parser, MapperService mapperService, String fieldName,
        SuggestionSearchContext.SuggestionContext suggestion, ParseFieldMatcher parseFieldMatcher) throws IOException {
    
    if ("analyzer".equals(fieldName)) {
        String analyzerName = parser.text();
        Analyzer analyzer = mapperService.analysisService().analyzer(analyzerName);
        if (analyzer == null) {
            throw new IllegalArgumentException("Analyzer [" + analyzerName + "] doesn't exists");
        }
        suggestion.setAnalyzer(analyzer);
    } else if ("field".equals(fieldName)) {
        suggestion.setField(parser.text());
    } else if ("size".equals(fieldName)) {
        suggestion.setSize(parser.intValue());
    } else if (parseFieldMatcher.match(fieldName, Fields.SHARD_SIZE)) {
        suggestion.setShardSize(parser.intValue());
    } else {
       return false;
    }
    return true;
    
}
 
Example 11
Source Project: Elasticsearch   Source File: TermSuggestParser.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public SuggestionSearchContext.SuggestionContext parse(XContentParser parser, MapperService mapperService,
        IndexQueryParserService queryParserService, HasContextAndHeaders headersContext) throws IOException {
    XContentParser.Token token;
    String fieldName = null;
    TermSuggestionContext suggestion = new TermSuggestionContext(suggester);
    DirectSpellcheckerSettings settings = suggestion.getDirectSpellCheckerSettings();
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            fieldName = parser.currentName();
        } else if (token.isValue()) {
            parseTokenValue(parser, mapperService, fieldName, suggestion, settings, queryParserService.parseFieldMatcher());
        } else {
            throw new IllegalArgumentException("suggester[term]  doesn't support field [" + fieldName + "]");
        }
    }
    return suggestion;
}
 
Example 12
Source Project: Elasticsearch   Source File: IndexQueryParserService.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public IndexQueryParserService(Index index, IndexSettingsService indexSettingsService,
                               IndicesQueriesRegistry indicesQueriesRegistry,
                               ScriptService scriptService, AnalysisService analysisService,
                               MapperService mapperService, IndexCache indexCache, IndexFieldDataService fieldDataService,
                               BitsetFilterCache bitsetFilterCache,
                               @Nullable SimilarityService similarityService) {
    super(index, indexSettingsService.getSettings());
    this.indexSettingsService = indexSettingsService;
    this.scriptService = scriptService;
    this.analysisService = analysisService;
    this.mapperService = mapperService;
    this.similarityService = similarityService;
    this.indexCache = indexCache;
    this.fieldDataService = fieldDataService;
    this.bitsetFilterCache = bitsetFilterCache;

    Settings indexSettings = indexSettingsService.getSettings();
    this.defaultField = indexSettings.get(DEFAULT_FIELD, AllFieldMapper.NAME);
    this.queryStringLenient = indexSettings.getAsBoolean(QUERY_STRING_LENIENT, false);
    this.parseFieldMatcher = new ParseFieldMatcher(indexSettings);
    this.defaultAllowUnmappedFields = indexSettings.getAsBoolean(ALLOW_UNMAPPED, true);
    this.indicesQueriesRegistry = indicesQueriesRegistry;
}
 
Example 13
Source Project: Elasticsearch   Source File: ParentChildIndexFieldData.java    License: Apache License 2.0 6 votes vote down vote up
public ParentChildIndexFieldData(Index index, Settings indexSettings, MappedFieldType.Names fieldNames,
                                 FieldDataType fieldDataType, IndexFieldDataCache cache, MapperService mapperService,
                                 CircuitBreakerService breakerService) {
    super(index, indexSettings, fieldNames, fieldDataType, cache);
    this.breakerService = breakerService;
    if (Version.indexCreated(indexSettings).before(Version.V_2_0_0_beta1)) {
        parentTypes = new TreeSet<>();
        for (DocumentMapper documentMapper : mapperService.docMappers(false)) {
            beforeCreate(documentMapper);
        }
        mapperService.addTypeListener(this);
    } else {
        ImmutableSortedSet.Builder<String> builder = ImmutableSortedSet.naturalOrder();
        for (DocumentMapper mapper : mapperService.docMappers(false)) {
            ParentFieldMapper parentFieldMapper = mapper.parentFieldMapper();
            if (parentFieldMapper.active()) {
                builder.add(parentFieldMapper.type());
            }
        }
        parentTypes = builder.build();
    }
}
 
Example 14
Source Project: crate   Source File: FieldsVisitor.java    License: Apache License 2.0 6 votes vote down vote up
public void postProcess(MapperService mapperService) {
    final Collection<String> types = mapperService.types();
    assert types.size() <= 1 : types;
    if (types.isEmpty() == false) {
        type = types.iterator().next();
    }
    for (Map.Entry<String, List<Object>> entry : fields().entrySet()) {
        MappedFieldType fieldType = mapperService.fullName(entry.getKey());
        if (fieldType == null) {
            throw new IllegalStateException("Field [" + entry.getKey()
                + "] exists in the index but not in mappings");
        }
        List<Object> fieldValues = entry.getValue();
        for (int i = 0; i < fieldValues.size(); i++) {
            fieldValues.set(i, fieldType.valueForDisplay(fieldValues.get(i)));
        }
    }
}
 
Example 15
Source Project: crate   Source File: CodecService.java    License: Apache License 2.0 6 votes vote down vote up
public CodecService(@Nullable MapperService mapperService, Logger logger) {
    final MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
    if (mapperService == null) {
        codecs.put(DEFAULT_CODEC, new Lucene84Codec());
        codecs.put(BEST_COMPRESSION_CODEC, new Lucene84Codec(Mode.BEST_COMPRESSION));
    } else {
        codecs.put(DEFAULT_CODEC,
            new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
        codecs.put(BEST_COMPRESSION_CODEC,
            new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
    }
    codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
    for (String codec : Codec.availableCodecs()) {
        codecs.put(codec, Codec.forName(codec));
    }
    this.codecs = codecs.immutableMap();
}
 
Example 16
Source Project: Elasticsearch   Source File: MapperTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static MapperService newMapperService(Path tempDir, Settings indexSettings, IndicesModule indicesModule) {
    Settings.Builder settingsBuilder = Settings.builder()
        .put("path.home", tempDir)
        .put(indexSettings);
    if (indexSettings.get(IndexMetaData.SETTING_VERSION_CREATED) == null) {
        settingsBuilder.put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT);
    }
    Settings settings = settingsBuilder.build();
    MapperRegistry mapperRegistry = indicesModule.getMapperRegistry();
    return new MapperService(new Index("test"),
        settings,
        newAnalysisService(settings),
        newSimilarityLookupService(settings),
        null,
        mapperRegistry,
        new DynamicArrayFieldMapperBuilderFactoryProvider());
}
 
Example 17
Source Project: Elasticsearch   Source File: MetaDataIndexUpgradeService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Checks the mappings for compatibility with the current version
 */
private void checkMappingsCompatibility(IndexMetaData indexMetaData) {
    Index index = new Index(indexMetaData.getIndex());
    Settings settings = indexMetaData.getSettings();
    try {
        SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, settings);
        // We cannot instantiate real analysis server at this point because the node might not have
        // been started yet. However, we don't really need real analyzers at this stage - so we can fake it
        try (AnalysisService analysisService = new FakeAnalysisService(index, settings)) {
            try (MapperService mapperService = new MapperService(index, settings, analysisService, similarityLookupService,
                    scriptService, mapperRegistry, dynamicArrayFieldMapperBuilderFactoryProvider)) {
                for (ObjectCursor<MappingMetaData> cursor : indexMetaData.getMappings().values()) {
                    MappingMetaData mappingMetaData = cursor.value;
                    mapperService.merge(mappingMetaData.type(), mappingMetaData.source(), MapperService.MergeReason.MAPPING_RECOVERY, false);
                }
            }
        }
    } catch (Exception ex) {
        // Wrap the inner exception so we have the index name in the exception message
        throw new IllegalStateException("unable to upgrade the mappings for the index [" + indexMetaData.getIndex() + "], reason: [" + ex.getMessage() + "]", ex);
    }
}
 
Example 18
Source Project: Elasticsearch   Source File: MappingUpdatedAction.java    License: Apache License 2.0 5 votes vote down vote up
private PutMappingRequestBuilder updateMappingRequest(String index, String type, Mapping mappingUpdate, final TimeValue timeout) {
    if (type.equals(MapperService.DEFAULT_MAPPING)) {
        throw new IllegalArgumentException("_default_ mapping should not be updated");
    }
    return client.preparePutMapping(index).setType(type).setSource(mappingUpdate.toString())
        .setMasterNodeTimeout(timeout).setTimeout(timeout);
}
 
Example 19
Source Project: crate   Source File: MetaDataCreateIndexService.java    License: Apache License 2.0 5 votes vote down vote up
static IndexMetaData validateResize(ClusterState state, String sourceIndex,
                                       Set<String> targetIndexMappingsTypes, String targetIndexName,
                                       Settings targetIndexSettings) {
    if (state.metaData().hasIndex(targetIndexName)) {
        throw new ResourceAlreadyExistsException(state.metaData().index(targetIndexName).getIndex());
    }
    final IndexMetaData sourceMetaData = state.metaData().index(sourceIndex);
    if (sourceMetaData == null) {
        throw new IndexNotFoundException(sourceIndex);
    }
    // ensure index is read-only
    if (state.blocks().indexBlocked(ClusterBlockLevel.WRITE, sourceIndex) == false) {
        throw new IllegalStateException("index " + sourceIndex + " must be read-only to resize index. use \"index.blocks.write=true\"");
    }

    if ((targetIndexMappingsTypes.size() > 1 ||
        (targetIndexMappingsTypes.isEmpty() || targetIndexMappingsTypes.contains(MapperService.DEFAULT_MAPPING)) == false)) {
        throw new IllegalArgumentException("mappings are not allowed when resizing indices" +
            ", all mappings are copied from the source index");
    }

    if (IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.exists(targetIndexSettings)) {
        // this method applies all necessary checks ie. if the target shards are less than the source shards
        // of if the source shards are divisible by the number of target shards
        IndexMetaData.getRoutingFactor(sourceMetaData.getNumberOfShards(),
            IndexMetaData.INDEX_NUMBER_OF_SHARDS_SETTING.get(targetIndexSettings));
    }
    return sourceMetaData;
}
 
Example 20
Source Project: Elasticsearch   Source File: CollectorContext.java    License: Apache License 2.0 5 votes vote down vote up
public CollectorContext(MapperService mapperService,
                        IndexFieldDataService fieldData,
                        CollectorFieldsVisitor visitor,
                        int jobSearchContextId) {
    this.mapperService = mapperService;
    this.fieldData = fieldData;
    fieldsVisitor = visitor;
    this.jobSearchContextId = jobSearchContextId;
}
 
Example 21
Source Project: Elasticsearch   Source File: LuceneQueryBuilder.java    License: Apache License 2.0 5 votes vote down vote up
private static GeoPointFieldMapper.GeoPointFieldType getGeoPointFieldType(String fieldName, MapperService mapperService) {
    MappedFieldType fieldType =  mapperService.smartNameFieldType(fieldName);
    if (fieldType == null) {
        throw new IllegalArgumentException(String.format("column \"%s\" doesn't exist", fieldName));
    }
    if (!(fieldType instanceof GeoPointFieldMapper.GeoPointFieldType)) {
        throw new IllegalArgumentException(String.format("column \"%s\" isn't of type geo_point", fieldName));
    }
    return (GeoPointFieldMapper.GeoPointFieldType) fieldType;
}
 
Example 22
Source Project: Elasticsearch   Source File: TermSuggestParser.java    License: Apache License 2.0 5 votes vote down vote up
private void parseTokenValue(XContentParser parser, MapperService mapperService, String fieldName, TermSuggestionContext suggestion,
        DirectSpellcheckerSettings settings, ParseFieldMatcher parseFieldMatcher) throws IOException {
    if (!(SuggestUtils.parseSuggestContext(parser, mapperService, fieldName, suggestion, parseFieldMatcher) || SuggestUtils.parseDirectSpellcheckerSettings(
            parser, fieldName, settings, parseFieldMatcher))) {
        throw new IllegalArgumentException("suggester[term] doesn't support [" + fieldName + "]");

    }
}
 
Example 23
Source Project: crate   Source File: LuceneChangesSnapshot.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new "translog" snapshot from Lucene for reading operations whose seq# in the specified range.
 *
 * @param engineSearcher    the internal engine searcher which will be taken over if the snapshot is opened successfully
 * @param mapperService     the mapper service which will be mainly used to resolve the document's type and uid
 * @param searchBatchSize   the number of documents should be returned by each search
 * @param fromSeqNo         the min requesting seq# - inclusive
 * @param toSeqNo           the maximum requesting seq# - inclusive
 * @param requiredFullRange if true, the snapshot will strictly check for the existence of operations between fromSeqNo and toSeqNo
 */
LuceneChangesSnapshot(Engine.Searcher engineSearcher, MapperService mapperService, int searchBatchSize,
                      long fromSeqNo, long toSeqNo, boolean requiredFullRange) throws IOException {
    if (fromSeqNo < 0 || toSeqNo < 0 || fromSeqNo > toSeqNo) {
        throw new IllegalArgumentException("Invalid range; from_seqno [" + fromSeqNo + "], to_seqno [" + toSeqNo + "]");
    }
    if (searchBatchSize <= 0) {
        throw new IllegalArgumentException("Search_batch_size must be positive [" + searchBatchSize + "]");
    }
    final AtomicBoolean closed = new AtomicBoolean();
    this.onClose = () -> {
        if (closed.compareAndSet(false, true)) {
            IOUtils.close(engineSearcher);
        }
    };
    this.mapperService = mapperService;
    final long requestingSize = (toSeqNo - fromSeqNo) == Long.MAX_VALUE ? Long.MAX_VALUE : (toSeqNo - fromSeqNo + 1L);
    this.searchBatchSize = requestingSize < searchBatchSize ? Math.toIntExact(requestingSize) : searchBatchSize;
    this.fromSeqNo = fromSeqNo;
    this.toSeqNo = toSeqNo;
    this.lastSeenSeqNo = fromSeqNo - 1;
    this.requiredFullRange = requiredFullRange;
    this.indexSearcher = new IndexSearcher(Lucene.wrapAllDocsLive(engineSearcher.getDirectoryReader()));
    this.indexSearcher.setQueryCache(null);
    this.parallelArray = new ParallelArray(this.searchBatchSize);
    final TopDocs topDocs = searchOperations(null);
    this.totalHits = Math.toIntExact(topDocs.totalHits.value);
    this.scoreDocs = topDocs.scoreDocs;
    fillParallelArray(scoreDocs, parallelArray);
}
 
Example 24
Source Project: crate   Source File: InternalEngine.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the estimated number of history operations whose seq# at least the provided seq# in this engine.
 */
@Override
public int estimateNumberOfHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        try (Translog.Snapshot snapshot = newChangesSnapshot(source, mapperService, Math.max(0, startingSeqNo), Long.MAX_VALUE, false)) {
            return snapshot.totalOperations();
        }
    } else {
        return getTranslog().estimateTotalOperationsFromMinSeq(startingSeqNo);
    }
}
 
Example 25
public static MapperService newMapperService(Settings settings, Client client) {
    Settings indexSettings = Settings.builder()
            .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
            .put("path.home", System.getProperty("path.home"))
            .put("client.type", "node")
            .put(settings)
            .build();
    Index index = new Index("test");
    Injector parentInjector = new ModulesBuilder()
            .add(new SettingsModule(indexSettings),
            new EnvironmentModule(new Environment(indexSettings)))
            .createInjector();
    AnalysisModule analysisModule = new AnalysisModule(indexSettings,
            parentInjector.getInstance(IndicesAnalysisService.class));
    new AnalysisBaseformPlugin(settings).onModule(analysisModule);
    Injector injector = new ModulesBuilder().add(new IndexSettingsModule(index, indexSettings),
            new IndexNameModule(index),
            analysisModule)
            .createChildInjector(parentInjector);
    AnalysisService analysisService = injector.getInstance(AnalysisService.class);
    SimilarityLookupService similarityLookupService = new SimilarityLookupService(index, indexSettings);
    Map<String, Mapper.TypeParser> mappers = registerBuiltInMappers();
    Map<String, MetadataFieldMapper.TypeParser> metadataMappers = registerBuiltInMetadataMappers();
    MapperRegistry mapperRegistry = new MapperRegistry(mappers, metadataMappers);
    return new MapperService(new Index("test"),
            indexSettings,
            analysisService,
            similarityLookupService,
            null,
            mapperRegistry);
}
 
Example 26
Source Project: crate   Source File: IndexModule.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * creates a new mapper service to do administrative work like mapping updates. This *should not* be used for document parsing.
 * doing so will result in an exception.
 */
public MapperService newIndexMapperService(NamedXContentRegistry xContentRegistry, MapperRegistry mapperRegistry) throws IOException {
    return new MapperService(
        indexSettings,
        analysisRegistry.build(indexSettings),
        xContentRegistry,
        mapperRegistry,
        () -> {
            throw new UnsupportedOperationException("no index query shard context available");
        }
    );
}
 
Example 27
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    return new GeoPointDVIndexFieldData(index, fieldType.names(), fieldType.fieldDataType(),
            Version.indexCreated(indexSettings).before(Version.V_2_2_0));
}
 
Example 28
Source Project: crate   Source File: LuceneQueryBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public Context convert(Symbol query,
                       TransactionContext txnCtx,
                       MapperService mapperService,
                       String indexName,
                       QueryShardContext queryShardContext,
                       DocTableInfo table,
                       IndexCache indexCache) throws UnsupportedFeatureException {
    var refResolver = new LuceneReferenceResolver(
        indexName,
        mapperService::fullName,
        table.partitionedByColumns()
    );
    var normalizer = new EvaluatingNormalizer(functions, RowGranularity.PARTITION, refResolver, null);
    Context ctx = new Context(
        txnCtx,
        functions,
        mapperService,
        indexCache,
        queryShardContext,
        indexName,
        table.partitionedByColumns()
    );
    CoordinatorTxnCtx coordinatorTxnCtx = CoordinatorTxnCtx.systemTransactionContext();
    ctx.query = eliminateNullsIfPossible(
        inverseSourceLookup(normalizer.normalize(query, coordinatorTxnCtx)),
        s -> normalizer.normalize(s, coordinatorTxnCtx)
    ).accept(VISITOR, ctx);
    if (LOGGER.isTraceEnabled()) {
        LOGGER.trace("WHERE CLAUSE [{}] -> LUCENE QUERY [{}] ", query.toString(Style.UNQUALIFIED), ctx.query);
    }
    return ctx;
}
 
Example 29
Source Project: Elasticsearch   Source File: BytesBinaryDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache,
                               CircuitBreakerService breakerService, MapperService mapperService) {
    // Ignore breaker
    final Names fieldNames = fieldType.names();
    return new BytesBinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType());
}
 
Example 30
Source Project: crate   Source File: InternalEngine.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new history snapshot for reading operations since the provided seqno.
 * The returned snapshot can be retrieved from either Lucene index or translog files.
 */
@Override
public Translog.Snapshot readHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
    if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
        return newChangesSnapshot(source, mapperService, Math.max(0, startingSeqNo), Long.MAX_VALUE, false);
    } else {
        return getTranslog().newSnapshotFromMinSeqNo(startingSeqNo);
    }
}