org.elasticsearch.action.admin.indices.stats.IndexStats Java Examples

The following examples show how to use org.elasticsearch.action.admin.indices.stats.IndexStats. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: PrometheusMetricsCollector.java From elasticsearch-prometheus-exporter with Apache License 2.0

6 votes

private void updatePerIndexMetrics(ClusterHealthResponse chr, IndicesStatsResponse isr) {

        if (chr != null && isr != null) {
            for (Map.Entry<String, IndexStats> entry : isr.getIndices().entrySet()) {
                String indexName = entry.getKey();
                ClusterIndexHealth cih = chr.getIndices().get(indexName);
                catalog.setClusterGauge("index_status", cih.getStatus().value(), indexName);
                catalog.setClusterGauge("index_replicas_number", cih.getNumberOfReplicas(), indexName);
                catalog.setClusterGauge("index_shards_number", cih.getActiveShards(), "active", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getNumberOfShards(), "shards", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getActivePrimaryShards(), "active_primary", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getInitializingShards(), "initializing", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getRelocatingShards(), "relocating", indexName);
                catalog.setClusterGauge("index_shards_number", cih.getUnassignedShards(), "unassigned", indexName);
                IndexStats indexStats = entry.getValue();
                updatePerIndexContextMetrics(indexName, "total", indexStats.getTotal());
                updatePerIndexContextMetrics(indexName, "primaries", indexStats.getPrimaries());
            }
        }
    }

Example #2

Source File: ElasticsearchIndexManager.java From Raigad with Apache License 2.0

6 votes

void preCreateIndex(Client client, IndexMetadata indexMetadata, DateTime dateTime) throws UnsupportedAutoIndexException {
    logger.info("Pre-creating indices for {}*", indexMetadata.getIndexNamePattern());

    IndicesStatsResponse indicesStatsResponse = getIndicesStatsResponse(client);
    Map<String, IndexStats> indexStatsMap = indicesStatsResponse.getIndices();

    if (indexStatsMap == null || indexStatsMap.isEmpty()) {
        logger.info("No existing indices, no need to pre-create");
        return;
    }

    indexStatsMap.keySet().stream()
        .filter(indexName -> indexMetadata.getIndexNameFilter().filter(indexName))
        .findFirst()
        .ifPresent(indexName -> {
            try {
                createIndex(client, indexMetadata.getIndexNameToPreCreate(dateTime));
            } catch (UnsupportedAutoIndexException e) {
                logger.error("Invalid index metadata: " + indexMetadata.toString(), e);
            }
        });
}

Example #3

Source File: TestElasticsearchIndexManager.java From Raigad with Apache License 2.0

6 votes

@Test
public void testRunIndexManagement_NotActionable_NoIndex() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 20}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}

Example #4

Source File: TestElasticsearchIndexManager.java From Raigad with Apache License 2.0

6 votes

@Test
public void testRunIndexManagement_NotActionable_NoRetentionPeriod() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(0)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}

Example #5

Source File: TestElasticsearchIndexManager.java From Raigad with Apache License 2.0

6 votes

@Test
public void testCheckIndexRetention_Overlapping() throws Exception {
    String serializedIndexMetadata = "[{\"preCreate\": false, \"retentionType\": \"hourly\", \"retentionPeriod\": 2, \"indexName\": \"nf_errors_log\"}," +
            "{\"preCreate\": false, \"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log201712\"}]";
    List<IndexMetadata> indexMetadataList = IndexUtils.parseIndexMetadata(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2017121110", new IndexStats("nf_errors_log2017121110", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121111", new IndexStats("nf_errors_log2017121111", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121112", new IndexStats("nf_errors_log2017121112", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121113", new IndexStats("nf_errors_log2017121113", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017121114", new IndexStats("nf_errors_log2017121114", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement(elasticsearchClient, indexMetadataList, new DateTime("2017-12-11T13:30Z"));

    verify(elasticsearchIndexManager, times(2)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));
}

Example #6

Source File: ElasticSearchHealthCheckJob.java From soundwave with Apache License 2.0

5 votes

private void logIndexStats(Map<String, IndexStats> indexStatsMap) {
  for (IndexStats stat : indexStatsMap.values()) {
    String indexKey = "esindex_" + stat.getIndex();
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryDocCount"),
        stat.getPrimaries().getDocs().getCount());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalDocCount"),
        stat.getTotal().getDocs().getCount());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "shardsCount"),
        stat.getShards().length);
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "primaryStoreSizeMB"),
        stat.getPrimaries().getStore().getSize().getMbFrac());
    Stats.setGauge(StatsUtil.getStatsName(indexKey, "totalStoreSizeMB"),
        stat.getTotal().getStore().getSize().getMbFrac());
  }
}

Example #7

Source File: ElasticsearchIndexManager.java From Raigad with Apache License 2.0

5 votes

void runIndexManagement(Client esTransportClient, List<IndexMetadata> indexMetadataList, DateTime dateTime) {
    // Find all the indices
    IndicesStatsResponse indicesStatsResponse = getIndicesStatsResponse(esTransportClient);
    Map<String, IndexStats> indexStatsMap = indicesStatsResponse.getIndices();

    if (indexStatsMap == null || indexStatsMap.isEmpty()) {
        logger.info("Cluster is empty, no indices found");
        return;
    }

    for (IndexMetadata indexMetadata : indexMetadataList) {
        if (!indexMetadata.isActionable()) {
            logger.warn(String.format("Index metadata %s is not actionable, skipping", indexMetadata));
            continue;
        }

        try {
            checkIndexRetention(esTransportClient, indexStatsMap.keySet(), indexMetadata, dateTime);

            if (indexMetadata.isPreCreate()) {
                preCreateIndex(esTransportClient, indexMetadata, dateTime);
            }
        } catch (Exception e) {
            logger.error("Caught an exception while building index metadata information from configuration property", e);
            return;
        }
    }
}

Example #8

Source File: TestElasticsearchIndexManager.java From Raigad with Apache License 2.0

5 votes

@Test
public void testRunIndexManagement() throws Exception {
    String serializedIndexMetadata = "[{\"retentionType\": \"yearly\", \"retentionPeriod\": 3, \"indexName\": \"nf_errors_log\"}]";
    when(config.getIndexMetadata()).thenReturn(serializedIndexMetadata);

    Map<String, IndexStats> indexStats = new HashMap<>();
    indexStats.put("nf_errors_log2018", new IndexStats("nf_errors_log2018", new ShardStats[]{}));
    indexStats.put("nf_errors_log2017", new IndexStats("nf_errors_log2017", new ShardStats[]{}));
    indexStats.put("nf_errors_log2016", new IndexStats("nf_errors_log2016", new ShardStats[]{}));
    indexStats.put("nf_errors_log2015", new IndexStats("nf_errors_log2015", new ShardStats[]{}));
    indexStats.put("nf_errors_log2014", new IndexStats("nf_errors_log2014", new ShardStats[]{}));
    indexStats.put("nf_errors_log2013", new IndexStats("nf_errors_log2013", new ShardStats[]{}));
    indexStats.put("nf_errors_log2012", new IndexStats("nf_errors_log2012", new ShardStats[]{}));

    IndicesStatsResponse indicesStatsResponse = mock(IndicesStatsResponse.class);
    when(indicesStatsResponse.getIndices()).thenReturn(indexStats);

    doReturn(indicesStatsResponse).when(elasticsearchIndexManager).getIndicesStatsResponse(elasticsearchClient);

    elasticsearchIndexManager.runIndexManagement();

    verify(elasticsearchIndexManager, times(1)).checkIndexRetention(any(Client.class), anySet(), any(IndexMetadata.class), any(DateTime.class));

    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2012"), eq(AUTO_CREATE_INDEX_TIMEOUT));
    verify(elasticsearchIndexManager, times(1)).deleteIndices(any(Client.class), eq("nf_errors_log2013"), eq(AUTO_CREATE_INDEX_TIMEOUT));

    verify(elasticsearchIndexManager, times(0)).preCreateIndex(any(Client.class), any(IndexMetadata.class), any(DateTime.class));
}

Example #9

Source File: EsInstanceStore.java From soundwave with Apache License 2.0

4 votes

public Map<String, IndexStats> getIndexStats() throws Exception {
  IndicesStatsResponse resp = esClient.admin().indices().prepareStats().all().get();
  return resp.getIndices();
}

Example #10

Source File: IndexShardStatsResponse.java From Elasticsearch with Apache License 2.0

4 votes

public IndexStats getIndex(String index) {
    return getIndices().get(index);
}

Example #11

Source File: IndexShardStatsResponse.java From Elasticsearch with Apache License 2.0

4 votes

@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
    String level = params.param("level", "indices");
    boolean isLevelValid = "indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level) || "cluster".equalsIgnoreCase(level);
    if (!isLevelValid) {
        return builder;
    }

    if ("indices".equalsIgnoreCase(level) || "shards".equalsIgnoreCase(level)) {
        builder.startObject(Fields.INDICES);
        for (IndexStats indexStats : getIndices().values()) {
            builder.startObject(indexStats.getIndex(), XContentBuilder.FieldCaseConversion.NONE);

            builder.startObject("primaries");
            indexStats.getPrimaries().toXContent(builder, params);
            builder.endObject();

            builder.startObject("total");
            indexStats.getTotal().toXContent(builder, params);
            builder.endObject();

            if ("shards".equalsIgnoreCase(level)) {
                builder.startObject(Fields.SHARDS);
                for (IndexShardStats indexShardStats : indexStats) {
                    builder.startArray(Integer.toString(indexShardStats.getShardId().id()));
                    for (ShardStats shardStats : indexShardStats) {
                        builder.startObject();
                        shardStats.toXContent(builder, params);
                        builder.endObject();
                    }
                    builder.endArray();
                }
                builder.endObject();
            }
            builder.endObject();
        }
        builder.endObject();
    }

    return builder;
}