Java Code Examples for org.elasticsearch.index.Index

The following examples show how to use org.elasticsearch.index.Index. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: crate   Source File: SwapRelationsOperation.java    License: Apache License 2.0 6 votes vote down vote up
private void removeOccurrences(ClusterState state,
                               ClusterBlocks.Builder blocksBuilder,
                               RoutingTable.Builder routingBuilder,
                               MetaData.Builder updatedMetaData,
                               RelationName name) {
    String aliasOrIndexName = name.indexNameOrAlias();
    String templateName = PartitionName.templateName(name.schema(), name.name());
    for (Index index : indexNameResolver.concreteIndices(
        state, IndicesOptions.LENIENT_EXPAND_OPEN, aliasOrIndexName)) {

        String indexName = index.getName();
        routingBuilder.remove(indexName);
        updatedMetaData.remove(indexName);
        blocksBuilder.removeIndexBlocks(indexName);
    }
    updatedMetaData.removeTemplate(templateName);
}
 
Example 2
Source Project: crate   Source File: TransportShardDeleteActionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void prepare() throws Exception {
    indexUUID = UUIDs.randomBase64UUID();
    IndicesService indicesService = mock(IndicesService.class);
    IndexService indexService = mock(IndexService.class);
    when(indicesService.indexServiceSafe(new Index(TABLE_IDENT.indexNameOrAlias(), indexUUID))).thenReturn(indexService);
    indexShard = mock(IndexShard.class);
    when(indexService.getShard(0)).thenReturn(indexShard);


    transportShardDeleteAction = new TransportShardDeleteAction(
        MockTransportService.createNewService(
            Settings.EMPTY, Version.CURRENT, THREAD_POOL, clusterService.getClusterSettings()),
        mock(IndexNameExpressionResolver.class),
        mock(ClusterService.class),
        indicesService,
        mock(ThreadPool.class),
        mock(ShardStateAction.class),
        mock(SchemaUpdateClient.class)
    );
}
 
Example 3
Source Project: Elasticsearch   Source File: NodeIndicesStats.java    License: Apache License 2.0 6 votes vote down vote up
private Map<Index, CommonStats> createStatsByIndex() {
    Map<Index, CommonStats> statsMap = Maps.newHashMap();
    for (Map.Entry<Index, List<IndexShardStats>> entry : statsByShard.entrySet()) {
        if (!statsMap.containsKey(entry.getKey())) {
            statsMap.put(entry.getKey(), new CommonStats());
        }

        for (IndexShardStats indexShardStats : entry.getValue()) {
            for (ShardStats shardStats : indexShardStats.getShards()) {
                statsMap.get(entry.getKey()).add(shardStats.getStats());
            }
        }
    }

    return statsMap;
}
 
Example 4
public void testNormalization() throws Exception {
    Settings settings = Settings.builder()
            .put("index.analysis.analyzer.myAnalyzer.type", "icu_collation")
            .put("index.analysis.analyzer.myAnalyzer.language", "tr")
            .put("index.analysis.analyzer.myAnalyzer.strength", "primary")
            .put("index.analysis.analyzer.myAnalyzer.decomposition", "canonical")
            .build();
    ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
            settings,
            new BundlePlugin(Settings.EMPTY));
    Analyzer analyzer = analysis.indexAnalyzers.get("myAnalyzer");
    TokenStream tsUpper = analyzer.tokenStream(null, "I W\u0049\u0307LL USE TURKİSH CASING");
    BytesRef b1 = bytesFromTokenStream(tsUpper);
    TokenStream tsLower = analyzer.tokenStream(null, "ı will use turkish casıng");
    BytesRef b2 = bytesFromTokenStream(tsLower);
    assertTrue(compare(b1.bytes, b2.bytes) == 0);
}
 
Example 5
Source Project: Elasticsearch   Source File: NodeEnvironment.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tries to find all allocated shards for the given index
 * on the current node. NOTE: This methods is prone to race-conditions on the filesystem layer since it might not
 * see directories created concurrently or while it's traversing.
 * @param index the index to filter shards
 * @return a set of shard IDs
 * @throws IOException if an IOException occurs
 */
public Set<ShardId> findAllShardIds(final Index index) throws IOException {
    assert index != null;
    if (nodePaths == null || locks == null) {
        throw new IllegalStateException("node is not configured to store local location");
    }
    assert assertEnvIsLocked();
    final Set<ShardId> shardIds = Sets.newHashSet();
    String indexName = index.name();
    for (final NodePath nodePath : nodePaths) {
        Path location = nodePath.indicesPath;
        if (Files.isDirectory(location)) {
            try (DirectoryStream<Path> indexStream = Files.newDirectoryStream(location)) {
                for (Path indexPath : indexStream) {
                    if (indexName.equals(indexPath.getFileName().toString())) {
                        shardIds.addAll(findAllShardsForIndex(indexPath));
                    }
                }
            }
        }
    }
    return shardIds;
}
 
Example 6
Source Project: crate   Source File: TransportUpdateSettingsAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void masterOperation(final UpdateSettingsRequest request, final ClusterState state, final ActionListener<AcknowledgedResponse> listener) {
    final Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request);
    UpdateSettingsClusterStateUpdateRequest clusterStateUpdateRequest = new UpdateSettingsClusterStateUpdateRequest()
            .indices(concreteIndices)
            .settings(request.settings())
            .setPreserveExisting(request.isPreserveExisting())
            .ackTimeout(request.timeout())
            .masterNodeTimeout(request.masterNodeTimeout());

    updateSettingsService.updateSettings(clusterStateUpdateRequest, new ActionListener<ClusterStateUpdateResponse>() {
        @Override
        public void onResponse(ClusterStateUpdateResponse response) {
            listener.onResponse(new AcknowledgedResponse(response.isAcknowledged()));
        }

        @Override
        public void onFailure(Exception t) {
            logger.debug(() -> new ParameterizedMessage("failed to update settings on indices [{}]", (Object) concreteIndices), t);
            listener.onFailure(t);
        }
    });
}
 
Example 7
Source Project: Elasticsearch   Source File: IndexStore.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public IndexStore(Index index, IndexSettingsService settingsService, IndicesStore indicesStore) {
    super(index, settingsService.getSettings());
    Settings indexSettings = settingsService.getSettings();
    this.indicesStore = indicesStore;

    this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none");
    if (rateLimitingType.equalsIgnoreCase("node")) {
        nodeRateLimiting = true;
    } else {
        nodeRateLimiting = false;
        rateLimiting.setType(rateLimitingType);
    }
    this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0));
    rateLimiting.setMaxRate(rateLimitingThrottle);

    logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle);
    this.settingsService = settingsService;
    this.settingsService.addListener(applySettings);
}
 
Example 8
Source Project: Elasticsearch   Source File: CodecService.java    License: Apache License 2.0 6 votes vote down vote up
private CodecService(Index index, Settings indexSettings, MapperService mapperService) {
    super(index, indexSettings);
    this.mapperService = mapperService;
    MapBuilder<String, Codec> codecs = MapBuilder.<String, Codec>newMapBuilder();
    if (mapperService == null) {
        codecs.put(DEFAULT_CODEC, new Lucene54Codec());
        codecs.put(BEST_COMPRESSION_CODEC, new Lucene54Codec(Mode.BEST_COMPRESSION));
    } else {
        codecs.put(DEFAULT_CODEC, 
                new PerFieldMappingPostingFormatCodec(Mode.BEST_SPEED, mapperService, logger));
        codecs.put(BEST_COMPRESSION_CODEC, 
                new PerFieldMappingPostingFormatCodec(Mode.BEST_COMPRESSION, mapperService, logger));
    }
    codecs.put(LUCENE_DEFAULT_CODEC, Codec.getDefault());
    for (String codec : Codec.availableCodecs()) {
        codecs.put(codec, Codec.forName(codec));
    }
    this.codecs = codecs.immutableMap();
}
 
Example 9
Source Project: Elasticsearch   Source File: HunspellTokenFilterFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Inject
public HunspellTokenFilterFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name, @Assisted Settings settings, HunspellService hunspellService)  {
    super(index, indexSettingsService.getSettings(), name, settings);

    String locale = settings.get("locale", settings.get("language", settings.get("lang", null)));
    if (locale == null) {
        throw new IllegalArgumentException("missing [locale | language | lang] configuration for hunspell token filter");
    }

    dictionary = hunspellService.getDictionary(locale);
    if (dictionary == null) {
        throw new IllegalArgumentException(String.format(Locale.ROOT, "Unknown hunspell dictionary for locale [%s]", locale));
    }

    dedup = settings.getAsBoolean("dedup", true);
    longestOnly = settings.getAsBoolean("longest_only", false);
}
 
Example 10
Source Project: crate   Source File: InternalBlobTableInfoFactory.java    License: Apache License 2.0 5 votes vote down vote up
private IndexMetaData resolveIndexMetaData(String tableName, ClusterState state) {
    String indexName = BlobIndex.fullIndexName(tableName);
    Index index;
    try {
        index = indexNameExpressionResolver.concreteIndices(state, IndicesOptions.strictExpandOpen(), indexName)[0];
    } catch (IndexNotFoundException ex) {
        throw new RelationUnknown(indexName, ex);
    }
    return state.metaData().index(index);
}
 
Example 11
Source Project: crate   Source File: ShardId.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Parse the string representation of this shardId back to an object.
 * We lose index uuid information here, but since we use toString in
 * rest responses, this is the best we can do to reconstruct the object
 * on the client side.
 */
public static ShardId fromString(String shardIdString) {
    int splitPosition = shardIdString.indexOf("][");
    if (splitPosition <= 0 || shardIdString.charAt(0) != '[' || shardIdString.charAt(shardIdString.length() - 1) != ']') {
        throw new IllegalArgumentException("Unexpected shardId string format, expected [indexName][shardId] but got " + shardIdString);
    }
    String indexName = shardIdString.substring(1, splitPosition);
    int shardId = Integer.parseInt(shardIdString.substring(splitPosition + 2, shardIdString.length() - 1));
    return new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), shardId);
}
 
Example 12
Source Project: crate   Source File: MetaStateService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Writes the index state.
 * <p>
 * This method is public for testing purposes.
 *
 * @throws WriteStateException if exception when writing state occurs. {@link WriteStateException#isDirty()} will always return
 *                             false, because new index state file is not yet referenced by manifest file.
 */
public long writeIndex(String reason, IndexMetaData indexMetaData) throws WriteStateException {
    final Index index = indexMetaData.getIndex();
    LOGGER.trace("[{}] writing state, reason [{}]", index, reason);
    try {
        long generation = INDEX_META_DATA_FORMAT.write(indexMetaData,
                nodeEnv.indexPaths(indexMetaData.getIndex()));
        LOGGER.trace("[{}] state written", index);
        return generation;
    } catch (WriteStateException ex) {
        throw new WriteStateException(false, "[" + index + "]: failed to write index state", ex);
    }
}
 
Example 13
public void testFoldingAnalyzerWithExceptions() throws Exception {
    String resource = "icu_normalize.json";
    Settings settings = Settings.builder()
            .loadFromStream(resource, getClass().getResourceAsStream(resource), true)
            .build();
    ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
            settings,
            new BundlePlugin(Settings.EMPTY));
    Analyzer analyzer = analysis.indexAnalyzers.get("my_icu_analyzer_with_exceptions");
    TokenStream ts = analyzer.tokenStream("test", "Jörg Prante");
    String[] expected = { "jörg", "prante" };
    assertTokenStreamContents(ts, expected);
}
 
Example 14
Source Project: crate   Source File: MetaStateService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Loads the full state, which includes both the global state and all the indices meta data. <br>
 * When loading, manifest file is consulted (represented by {@link Manifest} class), to load proper generations. <br>
 * If there is no manifest file on disk, this method fallbacks to BWC mode, where latest generation of global and indices
 * metadata is loaded. Please note that currently there is no way to distinguish between manifest file being removed and manifest
 * file was not yet created. It means that this method always fallbacks to BWC mode, if there is no manifest file.
 *
 * @return tuple of {@link Manifest} and {@link MetaData} with global metadata and indices metadata. If there is no state on disk,
 * meta state with globalGeneration -1 and empty meta data is returned.
 * @throws IOException if some IOException when loading files occurs or there is no metadata referenced by manifest file.
 */
Tuple<Manifest, MetaData> loadFullState() throws IOException {
    final Manifest manifest = MANIFEST_FORMAT.loadLatestState(LOGGER, namedXContentRegistry, nodeEnv.nodeDataPaths());
    if (manifest == null) {
        return loadFullStateBWC();
    }

    final MetaData.Builder metaDataBuilder;
    if (manifest.isGlobalGenerationMissing()) {
        metaDataBuilder = MetaData.builder();
    } else {
        final MetaData globalMetaData = META_DATA_FORMAT.loadGeneration(LOGGER, namedXContentRegistry, manifest.getGlobalGeneration(),
                nodeEnv.nodeDataPaths());
        if (globalMetaData != null) {
            metaDataBuilder = MetaData.builder(globalMetaData);
        } else {
            throw new IOException("failed to find global metadata [generation: " + manifest.getGlobalGeneration() + "]");
        }
    }

    for (Map.Entry<Index, Long> entry : manifest.getIndexGenerations().entrySet()) {
        final Index index = entry.getKey();
        final long generation = entry.getValue();
        final String indexFolderName = index.getUUID();
        final IndexMetaData indexMetaData = INDEX_META_DATA_FORMAT.loadGeneration(LOGGER, namedXContentRegistry, generation,
                nodeEnv.resolveIndexFolder(indexFolderName));
        if (indexMetaData != null) {
            metaDataBuilder.put(indexMetaData, false);
        } else {
            throw new IOException("failed to find metadata for existing index " + index.getName() + " [location: " + indexFolderName +
                    ", generation: " + generation + "]");
        }
    }

    return new Tuple<>(manifest, metaDataBuilder.build());
}
 
Example 15
Source Project: Elasticsearch   Source File: AnalysisService.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public AnalysisService(Index index, IndexSettingsService indexSettingsService, @Nullable IndicesAnalysisService indicesAnalysisService,
                       @Nullable Map<String, AnalyzerProviderFactory> analyzerFactoryFactories,
                       @Nullable Map<String, TokenizerFactoryFactory> tokenizerFactoryFactories,
                       @Nullable Map<String, CharFilterFactoryFactory> charFilterFactoryFactories,
                       @Nullable Map<String, TokenFilterFactoryFactory> tokenFilterFactoryFactories) {
    this(index, indexSettingsService.getSettings(), indicesAnalysisService, analyzerFactoryFactories, tokenizerFactoryFactories,
            charFilterFactoryFactories, tokenFilterFactoryFactories);

}
 
Example 16
Source Project: Elasticsearch   Source File: InternalIndicesLifecycle.java    License: Apache License 2.0 5 votes vote down vote up
public void beforeIndexCreated(Index index, Settings indexSettings) {
    for (Listener listener : listeners) {
        try {
            listener.beforeIndexCreated(index, indexSettings);
        } catch (Throwable t) {
            logger.warn("[{}] failed to invoke before index created callback", t, index.name());
            throw t;
        }
    }
}
 
Example 17
public void testSeven() throws Exception {
    String source = "Procter & Gamble ist Procter&Gamble. Schwarz - weiss ist schwarz-weiss";

    String[] expected = {
            "Procter",
            "Gamble",
            "ist",
            "Procter&Gamble",
            "Schwarz",
            "weiss",
            "ist",
            "schwarz-weiss",
            "schwarzweiss",
            "weiss",
            "schwarz"
    };

    String resource = "hyphen_tokenizer.json";
    Settings settings = Settings.builder()
            .loadFromStream(resource, getClass().getResourceAsStream(resource), true)
            .build();
    ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
            settings,
            new BundlePlugin(Settings.EMPTY));
    Tokenizer tokenizer = analysis.tokenizer.get("my_hyphen_tokenizer").create();
    tokenizer.setReader(new StringReader(source));
    TokenFilterFactory tokenFilter = analysis.tokenFilter.get("hyphen");
    assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
}
 
Example 18
public void testReorder() throws Exception {
    String resource = "icu_collation.json";
    Settings settings = Settings.builder()
            .loadFromStream(resource, getClass().getResourceAsStream(resource), true)
            .build();
    ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
            settings,
            new BundlePlugin(Settings.EMPTY));
    Analyzer analyzer = analysis.indexAnalyzers.get("reorder");
    assertNotNull(analyzer);
}
 
Example 19
Source Project: crate   Source File: IndexMetaData.java    License: Apache License 2.0 5 votes vote down vote up
private IndexMetaData(Index index, long version, long mappingVersion, long settingsVersion, long[] primaryTerms, State state, int numberOfShards, int numberOfReplicas, Settings settings,
                      ImmutableOpenMap<String, MappingMetaData> mappings, ImmutableOpenMap<String, AliasMetaData> aliases,
                      ImmutableOpenMap<String, DiffableStringMap> customData, ImmutableOpenIntMap<Set<String>> inSyncAllocationIds,
                      DiscoveryNodeFilters requireFilters, DiscoveryNodeFilters initialRecoveryFilters, DiscoveryNodeFilters includeFilters, DiscoveryNodeFilters excludeFilters,
                      Version indexCreatedVersion, Version indexUpgradedVersion,
                      int routingNumShards, int routingPartitionSize, ActiveShardCount waitForActiveShards) {

    this.index = index;
    this.version = version;
    assert mappingVersion >= 0 : mappingVersion;
    this.mappingVersion = mappingVersion;
    assert settingsVersion >= 0 : settingsVersion;
    this.settingsVersion = settingsVersion;
    this.primaryTerms = primaryTerms;
    assert primaryTerms.length == numberOfShards;
    this.state = state;
    this.numberOfShards = numberOfShards;
    this.numberOfReplicas = numberOfReplicas;
    this.totalNumberOfShards = numberOfShards * (numberOfReplicas + 1);
    this.settings = settings;
    this.mappings = mappings;
    this.customData = customData;
    this.aliases = aliases;
    this.inSyncAllocationIds = inSyncAllocationIds;
    this.requireFilters = requireFilters;
    this.includeFilters = includeFilters;
    this.excludeFilters = excludeFilters;
    this.initialRecoveryFilters = initialRecoveryFilters;
    this.indexCreatedVersion = indexCreatedVersion;
    this.indexUpgradedVersion = indexUpgradedVersion;
    this.routingNumShards = routingNumShards;
    this.routingFactor = routingNumShards / numberOfShards;
    this.routingPartitionSize = routingPartitionSize;
    this.waitForActiveShards = waitForActiveShards;
    assert numberOfShards * routingFactor == routingNumShards : routingNumShards + " must be a multiple of " + numberOfShards;
}
 
Example 20
public void testUnicodeUsage() throws Exception {
    Settings settings = Settings.builder()
            .put("index.analysis.analyzer.myanalyzer.type", "sortform")
            .put("index.analysis.analyzer.myanalyzer.filter", "sortform")
            .build();
    ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
            settings,
            new BundlePlugin(Settings.EMPTY));
    Analyzer myanalyzer = analysis.indexAnalyzers.get("myanalyzer");
    // Unicode 0098: START OF STRING
    // Unicode 009C: STRING TERMINATOR
    assertAnalyzesTo(myanalyzer, "\u0098Der\u009c Titel des Buches", new String[]{"Titel des Buches"});
}
 
Example 21
Source Project: Elasticsearch   Source File: NodeIndicesStats.java    License: Apache License 2.0 5 votes vote down vote up
public NodeIndicesStats(CommonStats oldStats, Map<Index, List<IndexShardStats>> statsByShard) {
    //this.stats = stats;
    this.statsByShard = statsByShard;

    // make a total common stats from old ones and current ones
    this.stats = oldStats;
    for (List<IndexShardStats> shardStatsList : statsByShard.values()) {
        for (IndexShardStats indexShardStats : shardStatsList) {
            for (ShardStats shardStats : indexShardStats.getShards()) {
                stats.add(shardStats.getStats());
            }
        }
    }
}
 
Example 22
Source Project: Elasticsearch   Source File: LengthTokenFilterFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public LengthTokenFilterFactory(Index index, IndexSettingsService indexSettingsService, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    min = settings.getAsInt("min", 0);
    max = settings.getAsInt("max", Integer.MAX_VALUE);
    if (version.onOrAfter(Version.LUCENE_4_4) && settings.get(ENABLE_POS_INC_KEY) != null) {
        throw new IllegalArgumentException(ENABLE_POS_INC_KEY + " is not supported anymore. Please fix your analysis chain or use"
                + " an older compatibility version (<=4.3) but beware that it might cause highlighting bugs.");
    }
    enablePositionIncrements = version.onOrAfter(Version.LUCENE_4_4) ? true : settings.getAsBoolean(ENABLE_POS_INC_KEY, true);
}
 
Example 23
Source Project: crate   Source File: ClusterChangedEvent.java    License: Apache License 2.0 5 votes vote down vote up
private List<Index> indicesDeletedFromTombstones() {
    // We look at the full tombstones list to see which indices need to be deleted.  In the case of
    // a valid previous cluster state, indicesDeletedFromClusterState() will be used to get the deleted
    // list, so a diff doesn't make sense here.  When a node (re)joins the cluster, its possible for it
    // to re-process the same deletes or process deletes about indices it never knew about.  This is not
    // an issue because there are safeguards in place in the delete store operation in case the index
    // folder doesn't exist on the file system.
    List<IndexGraveyard.Tombstone> tombstones = state.metaData().indexGraveyard().getTombstones();
    return tombstones.stream().map(IndexGraveyard.Tombstone::getIndex).collect(Collectors.toList());
}
 
Example 24
Source Project: Elasticsearch   Source File: ChineseAnalyzerProvider.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public ChineseAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    // old index: best effort
    analyzer = new StandardAnalyzer();
    analyzer.setVersion(version);
    
}
 
Example 25
public void testDecompound() throws Exception {

        String source = "Die Jahresfeier der Rechtsanwaltskanzleien auf dem Donaudampfschiff hat viel Ökosteuer gekostet";

        String[] expected = {
                "Die",
                "Jahresfeier",
                "jahres",
                "feier",
                "der",
                "Rechtsanwaltskanzleien",
                "rechts",
                "anwalts",
                "kanzleien",
                "auf",
                "dem",
                "Donaudampfschiff",
                "donau",
                "dampf",
                "schiff",
                "hat",
                "viel",
                "Ökosteuer",
                "ökos",
                "teuer",
                "gekostet"
        };

        Settings settings = Settings.builder()
                .put("index.analysis.analyzer.myanalyzer.type", "custom")
                .put("index.analysis.analyzer.myanalyzer.tokenizer", "standard")
                .put("index.analysis.analyzer.myanalyzer.filter.0", "fst_decompound")
                .put("index.analysis.analyzer.myanalyzer.filter.1", "unique")
                .build();
        ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
                settings,
                new BundlePlugin(Settings.EMPTY), new CommonAnalysisPlugin());
        Analyzer myanalyzer = analysis.indexAnalyzers.get("myanalyzer");
        assertAnalyzesTo(myanalyzer, source, expected);
    }
 
Example 26
Source Project: Elasticsearch   Source File: TransportBulkAction.java    License: Apache License 2.0 5 votes vote down vote up
private boolean addFailureIfIndexIsUnavailable(DocumentRequest request, BulkRequest bulkRequest, AtomicArray<BulkItemResponse> responses, int idx,
                                          final ConcreteIndices concreteIndices,
                                          final MetaData metaData) {
    String concreteIndex = concreteIndices.getConcreteIndex(request.index());
    Exception unavailableException = null;
    if (concreteIndex == null) {
        try {
            concreteIndex = concreteIndices.resolveIfAbsent(request);
        } catch (IndexClosedException | IndexNotFoundException ex) {
            // Fix for issue where bulk request references an index that
            // cannot be auto-created see issue #8125
            unavailableException = ex;
        }
    }
    if (unavailableException == null) {
        IndexMetaData indexMetaData = metaData.index(concreteIndex);
        if (indexMetaData.getState() == IndexMetaData.State.CLOSE) {
            unavailableException = new IndexClosedException(new Index(metaData.index(request.index()).getIndex()));
        }
    }
    if (unavailableException != null) {
        BulkItemResponse.Failure failure = new BulkItemResponse.Failure(request.index(), request.type(), request.id(),
                unavailableException);
        String operationType = "unknown";
        if (request instanceof IndexRequest) {
            operationType = "index";
        } else if (request instanceof DeleteRequest) {
            operationType = "delete";
        } else if (request instanceof UpdateRequest) {
            operationType = "update";
        }
        BulkItemResponse bulkItemResponse = new BulkItemResponse(idx, operationType, failure);
        responses.set(idx, bulkItemResponse);
        // make sure the request gets never processed again
        bulkRequest.requests.set(idx, null);
        return true;
    }
    return false;
}
 
Example 27
Source Project: metron   Source File: ElasticsearchRequestSubmitterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void searchShouldHandleShardFailure() throws InvalidSearchException, IOException {
  // mocks
  SearchResponse response = mock(SearchResponse.class);
  SearchRequest request = new SearchRequest();
  ShardSearchFailure fail = mock(ShardSearchFailure.class);
  SearchShardTarget target = new SearchShardTarget("node1", mock(Index.class), 1, "metron");

  // response will have status of OK
  when(response.status()).thenReturn(RestStatus.OK);

  // response will indicate 1 search hit
  SearchHits hits = mock(SearchHits.class);
  when(hits.getTotalHits()).thenReturn(1L);

  // the response will report shard failures
  when(response.getFailedShards()).thenReturn(1);
  when(response.getTotalShards()).thenReturn(2);
  when(response.getHits()).thenReturn(hits);

  // the response will return the failures
  ShardSearchFailure[] failures = { fail };
  when(response.getShardFailures()).thenReturn(failures);

  // shard failure needs to report the node
  when(fail.shard()).thenReturn(target);

  // shard failure needs to report details of failure
  when(fail.index()).thenReturn("bro_index_2017-10-11");
  when(fail.shardId()).thenReturn(1);

  // search should succeed, even with failed shards
  ElasticsearchRequestSubmitter submitter = setup(response);
  SearchResponse actual = submitter.submitSearch(request);
  assertNotNull(actual);
}
 
Example 28
public void testGerman1() throws IOException {

        String source = "Ein schöner Tag in Köln im Café an der Straßenecke";

        String[] expected = {
            "Ein",
            "schoner",
            "Tag",
            "in",
            "Koln",
            "im",
            "Café",
            "an",
            "der",
            "Strassenecke"
        };
        String resource = "german_normalization_analysis.json";
        Settings settings = Settings.builder()
                .put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT)
                .put("path.home", System.getProperty("path.home"))
                .loadFromStream(resource, getClass().getResourceAsStream(resource), true)
                .build();
        ESTestCase.TestAnalysis analysis = ESTestCase.createTestAnalysis(new Index("test", "_na_"),
                settings,
                new BundlePlugin(Settings.EMPTY));

        TokenFilterFactory tokenFilter = analysis.tokenFilter.get("umlaut");
        Tokenizer tokenizer = analysis.tokenizer.get("standard").create();
        tokenizer.setReader(new StringReader(source));
        assertTokenStreamContents(tokenFilter.create(tokenizer), expected);
    }
 
Example 29
Source Project: Elasticsearch   Source File: ItalianAnalyzerProvider.java    License: Apache License 2.0 5 votes vote down vote up
@Inject
public ItalianAnalyzerProvider(Index index, IndexSettingsService indexSettingsService, Environment env, @Assisted String name, @Assisted Settings settings) {
    super(index, indexSettingsService.getSettings(), name, settings);
    analyzer = new ItalianAnalyzer(Analysis.parseStopWords(env, settings, ItalianAnalyzer.getDefaultStopSet()),
                                   Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET));
    analyzer.setVersion(version);
}
 
Example 30
Source Project: crate   Source File: IndicesService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This method returns true if the current node is allowed to delete the given index.
 * This is the case if the index is deleted in the metadata or there is no allocation
 * on the local node and the index isn't on a shared file system.
 * @param index {@code Index} to check whether deletion is allowed
 * @param indexSettings {@code IndexSettings} for the given index
 * @return true if the index can be deleted on this node
 */
public boolean canDeleteIndexContents(Index index, IndexSettings indexSettings) {
    // index contents can be deleted if its an already closed index (so all its resources have
    // already been relinquished)
    final IndexService indexService = indexService(index);
    if (indexService == null && nodeEnv.hasNodeFile()) {
        return true;
    }
    return false;
}