/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ package com.frontier45.flume.sink.elasticsearch2.client; import com.frontier45.flume.sink.elasticsearch2.ElasticSearchEventSerializer; import com.frontier45.flume.sink.elasticsearch2.IndexNameBuilder; import com.google.common.annotations.VisibleForTesting; import com.google.gson.Gson; import org.apache.flume.Context; import org.apache.flume.Event; import org.apache.flume.EventDeliveryException; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; import org.apache.http.client.methods.HttpPost; import org.apache.http.entity.StringEntity; import org.apache.http.impl.client.DefaultHttpClient; import org.apache.http.util.EntityUtils; import org.elasticsearch.common.bytes.BytesReference; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.Arrays; import java.util.HashMap; import java.util.Map; /** * Rest ElasticSearch client which is responsible for sending bulks of events to * ElasticSearch using ElasticSearch HTTP API. This is configurable, so any * config params required should be taken through this. */ public class ElasticSearchRestClient implements ElasticSearchClient { private static final String INDEX_OPERATION_NAME = "index"; private static final String INDEX_PARAM = "_index"; private static final String TYPE_PARAM = "_type"; private static final String TTL_PARAM = "_ttl"; private static final String BULK_ENDPOINT = "_bulk"; private static final Logger logger = LoggerFactory.getLogger(ElasticSearchRestClient.class); private final ElasticSearchEventSerializer serializer; private final RoundRobinList<String> serversList; private StringBuilder bulkBuilder; private HttpClient httpClient; public ElasticSearchRestClient(String[] hostNames, ElasticSearchEventSerializer serializer) { for (int i = 0; i < hostNames.length; ++i) { if (!hostNames[i].contains("http://") && !hostNames[i].contains("https://")) { hostNames[i] = "http://" + hostNames[i]; } } this.serializer = serializer; serversList = new RoundRobinList<String>(Arrays.asList(hostNames)); httpClient = new DefaultHttpClient(); bulkBuilder = new StringBuilder(); } @VisibleForTesting public ElasticSearchRestClient(String[] hostNames, ElasticSearchEventSerializer serializer, HttpClient client) { this(hostNames, serializer); httpClient = client; } @Override public void configure(Context context) { } @Override public void close() { } @Override public void addEvent(Event event, IndexNameBuilder indexNameBuilder, String indexType, long ttlMs) throws Exception { BytesReference content = serializer.getContentBuilder(event).bytes(); Map<String, Map<String, String>> parameters = new HashMap<String, Map<String, String>>(); Map<String, String> indexParameters = new HashMap<String, String>(); indexParameters.put(INDEX_PARAM, indexNameBuilder.getIndexName(event)); indexParameters.put(TYPE_PARAM, indexType); if (ttlMs > 0) { indexParameters.put(TTL_PARAM, Long.toString(ttlMs)); } parameters.put(INDEX_OPERATION_NAME, indexParameters); Gson gson = new Gson(); synchronized (bulkBuilder) { bulkBuilder.append(gson.toJson(parameters)); bulkBuilder.append("\n"); bulkBuilder.append(content.toBytesArray().toUtf8()); bulkBuilder.append("\n"); } } @Override public void execute() throws Exception { int statusCode = 0, triesCount = 0; HttpResponse response = null; String entity; synchronized (bulkBuilder) { entity = bulkBuilder.toString(); bulkBuilder = new StringBuilder(); } while (statusCode != HttpStatus.SC_OK && triesCount < serversList.size()) { triesCount++; String host = serversList.get(); String url = host + "/" + BULK_ENDPOINT; HttpPost httpRequest = new HttpPost(url); httpRequest.setEntity(new StringEntity(entity)); response = httpClient.execute(httpRequest); statusCode = response.getStatusLine().getStatusCode(); logger.info("Status code from elasticsearch: " + statusCode); if (response.getEntity() != null) logger.debug("Status message from elasticsearch: " + EntityUtils.toString(response.getEntity(), "UTF-8")); } if (statusCode != HttpStatus.SC_OK) { if (response != null && response.getEntity() != null) { throw new EventDeliveryException(EntityUtils.toString(response.getEntity(), "UTF-8")); } else { throw new EventDeliveryException("Elasticsearch status code was: " + statusCode); } } } }