Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder#add()
The following examples show how to use
org.elasticsearch.action.bulk.BulkRequestBuilder#add() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
| Source File: CassandraRiver.java From cassandra-river with Apache License 2.0 | 6 votes |
|
@Override
public void run() {
logger.info("Starting thread with {} keys", this.keys.rowColumnMap.size());
if (closed) {
return;
}
BulkRequestBuilder bulk = client.prepareBulk();
for(String key : this.keys.rowColumnMap.keySet()){
try {
String id = UUID.nameUUIDFromBytes(key.getBytes()).toString();
bulk.add(indexRequest(this.indexName).type(this.typeName)
.id(id)
.source(this.keys.rowColumnMap.get(key)));
} catch (Exception e) {
logger.error("failed to entry to bulk indexing");
}
if(bulk.numberOfActions() >= this.batchSize){
saveToEs(bulk);
bulk = client.prepareBulk();
}
}
}
Example 2
| Source File: BulkIndexEsTest.java From elastic-crud with Apache License 2.0 | 6 votes |
|
@Before
public void before() throws IOException {
final IndicesAdminClient indices = client.admin().indices();
if(!indices.prepareExists(INDEX).execute().actionGet().isExists()) {
indices.prepareCreate(INDEX).execute().actionGet();
}
final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
final BulkRequestBuilder bulk = client.prepareBulk();
for (int i = 0; i < SIZE; i++) {
final String name = UUID.randomUUID().toString();
final IndexRequest request = new IndexRequest(INDEX, TYPE);
request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
bulk.add(request);
}
client.bulk(bulk.request()).actionGet();
flush(INDEX);
}
Example 3
| Source File: ElasticSearch.java From hsweb-learning with Apache License 2.0 | 6 votes |
|
private static void BulkIndex(Client client) throws IOException {
BulkRequestBuilder requestBuilder = client.prepareBulk();
requestBuilder.add(client.prepareIndex("twitter","tweet","4")
.setSource(jsonBuilder()
.startObject()
.field("user","niekaijie")
.field("school","beiyou")
.field("address","haidianqu")
.endObject())
);
requestBuilder.add(client.prepareIndex("twitter","tweet","3")
.setSource(jsonBuilder()
.startObject()
.field("user","林志颖aa")
.field("school","台湾大学")
.field("address","台湾")
.endObject())
);
BulkResponse response = requestBuilder.get();
}
Example 4
| Source File: ElasticsearchTransportFactory.java From database-transform-tool with Apache License 2.0 | 6 votes |
|
@Override
public String bulkDelete(String index, String type, String... ids) {
try {
if(client==null){
init();
}
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (String id : ids) {
bulkRequest.add(client.prepareDelete(index, type, id));
}
BulkResponse result = bulkRequest.execute().get();
return result.toString();
}catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return null;
}
Example 5
| Source File: SpanServiceElasticsearch.java From hawkular-apm with Apache License 2.0 | 5 votes |
|
@Override
public void storeSpan(String tenantId, List<Span> spans, Function<Span, String> spanIdSupplier)
throws StoreException {
client.initTenant(tenantId);
BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();
for (Span span : spans) {
String json;
try {
json = serialize(span);
} catch(IOException ex){
log.errorf("Failed to serialize span %s", span);
throw new StoreException(ex);
}
log.tracef("Storing span: %s", json);
// modified id is used in index
final String modifiedId = spanIdSupplier.apply(span);
bulkRequestBuilder.add(client.getClient()
.prepareIndex(client.getIndex(tenantId), SPAN_TYPE, modifiedId)
.setSource(json));
}
BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();
if (bulkItemResponses.hasFailures()) {
log.tracef("Failed to store spans to elasticsearch: %s", bulkItemResponses.buildFailureMessage());
throw new StoreException(bulkItemResponses.buildFailureMessage());
}
log.trace("Success storing spans to elasticsearch");
}
Example 6
| Source File: ElasticsearchTemplate.java From summerframework with Apache License 2.0 | 5 votes |
|
public int updateBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
BulkRequestBuilder bulkRequest = esClient.prepareBulk();
for (String id : esBasicInfo.getIds()) {
bulkRequest.add(esClient.prepareUpdate(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
.setDoc(mapper.writeValueAsString(object), XContentType.JSON));
}
bulkRequest.execute().actionGet();
return bulkRequest.numberOfActions();
}
Example 7
| Source File: ElasticsearchDocumentWriter.java From CogStack-Pipeline with Apache License 2.0 | 5 votes |
|
@Override
public final void write(List<? extends Document> documents) throws Exception {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (Document doc : documents) {
XContentParser parser = null;
parser = XContentFactory.xContent(XContentType.JSON)
.createParser(NamedXContentRegistry.EMPTY,
DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
doc.getOutputData().getBytes());
parser.close();
XContentBuilder builder = jsonBuilder().copyCurrentStructure(parser);
IndexRequestBuilder request = client.prepareIndex(
indexName,
typeName).setSource(
builder);
request.setId(doc.getPrimaryKeyFieldValue());
bulkRequest.add(request);
}
//check that no nonessential processes failed
if (documents.size() != 0) {
BulkResponse response;
response = bulkRequest.execute().actionGet(timeout);
getResponses(response);
}
}
Example 8
| Source File: ElasticsearchTemplate.java From summerframework with Apache License 2.0 | 5 votes |
|
public int addBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
BulkRequestBuilder bulkRequest = esClient.prepareBulk();
for (String id : esBasicInfo.getIds()) {
bulkRequest.add(esClient.prepareIndex(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
.setSource(mapper.writeValueAsString(object), XContentType.JSON));
}
bulkRequest.execute().actionGet();
return bulkRequest.numberOfActions();
}
Example 9
| Source File: ElasticsearchTemplate.java From summerframework with Apache License 2.0 | 5 votes |
|
public int updateBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
BulkRequestBuilder bulkRequest = esClient.prepareBulk();
for (String id : esBasicInfo.getIds()) {
bulkRequest.add(esClient.prepareUpdate(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
.setDoc(mapper.writeValueAsString(object), XContentType.JSON));
}
bulkRequest.execute().actionGet();
return bulkRequest.numberOfActions();
}
Example 10
| Source File: ElasticsearchTemplate.java From summerframework with Apache License 2.0 | 5 votes |
|
public int deleteBatchData(ESBasicInfo esBasicInfo) {
BulkRequestBuilder bulkRequest = esClient.prepareBulk();
for (String id : esBasicInfo.getIds()) {
bulkRequest.add(esClient.prepareDelete(esBasicInfo.getIndex(), esBasicInfo.getType(), id));
}
BulkResponse response = bulkRequest.execute().actionGet();
log.info("status is:{}", response.status().getStatus());
return bulkRequest.numberOfActions();
}
Example 11
| Source File: TableMapStore.java From foxtrot with Apache License 2.0 | 5 votes |
|
@Override
public void storeAll(Map<String, Table> map) {
if(map == null) {
throw new TableMapStoreException("Illegal Store Request - Null Map");
}
if(map.containsKey(null)) {
throw new TableMapStoreException("Illegal Store Request - Null Key is Present");
}
logger.info("Store all called for multiple values");
BulkRequestBuilder bulkRequestBuilder = elasticsearchConnection.getClient()
.prepareBulk()
.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
for(Map.Entry<String, Table> mapEntry : map.entrySet()) {
try {
if(mapEntry.getValue() == null) {
throw new TableMapStoreException(
String.format("Illegal Store Request - Object is Null for Table - %s", mapEntry.getKey()));
}
Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(objectMapper, mapEntry.getValue());
bulkRequestBuilder.add(elasticsearchConnection.getClient()
.prepareIndex(TABLE_META_INDEX, TABLE_META_TYPE, mapEntry.getKey())
.setSource(sourceMap));
} catch (Exception e) {
throw new TableMapStoreException("Error bulk saving meta: ", e);
}
}
bulkRequestBuilder.execute()
.actionGet();
}
Example 12
| Source File: BulkDemo.java From elasticsearch-full with Apache License 2.0 | 5 votes |
|
@Test
public void testForClient() throws Exception {
BulkRequestBuilder bulkRequest = client.prepareBulk();
// either use client#prepare, or use Requests# to directly build index/delete requests
bulkRequest.add(client.prepareIndex("twitter", "tweet", "1")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", new Date())
.field("message", "trying out Elasticsearch")
.endObject()
)
);
bulkRequest.add(client.prepareIndex("twitter", "tweet", "2")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("user", "kimchy")
.field("postDate", new Date())
.field("message", "another post")
.endObject()
)
);
BulkResponse bulkResponse = bulkRequest.get();
if (bulkResponse.hasFailures()) {
// process failures by iterating through each bulk response item
}
}
Example 13
| Source File: SearchUpdateJobImpl.java From stash-codesearch-plugin with Apache License 2.0 | 5 votes |
|
@Override
public void doReindex(Client client, GitScm gitScm, GlobalSettings globalSettings) {
if (!globalSettings.getIndexingEnabled()) {
return;
}
deleteLatestIndexedNote(client);
while (true) {
try {
SearchRequestBuilder esReq = client.prepareSearch(ES_UPDATEALIAS)
.setSize(1000)
.setFetchSource(false)
.setRouting(getRepoDesc())
.setQuery(filteredQuery(matchAllQuery(), andFilter(
sfu.projectRepositoryFilter(
repository.getProject().getKey(), repository.getSlug()),
sfu.exactRefFilter(ref))));
BulkRequestBuilder bulkDelete = client.prepareBulk().setRefresh(true);
for (SearchHit hit : esReq.get().getHits().getHits()) {
bulkDelete.add(buildDeleteFromRef(client, hit.getType(), hit.getId()));
}
if (bulkDelete.numberOfActions() == 0) {
break;
}
bulkDelete.get();
} catch (Exception e) {
log.error("Could not delete documents for {}, aborting", toString(), e);
return;
}
}
doUpdate(client, gitScm, globalSettings);
}
Example 14
| Source File: EsAbstractBehavior.java From fess with Apache License 2.0 | 5 votes |
|
@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
if (esCb.getPreference() != null) {
esCb.setPreference(esCb.getPreference());
}
esCb.request().build(builder);
SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
String scrollId = response.getScrollId();
int count = 0;
try {
while (scrollId != null) {
final SearchHits searchHits = getSearchHits(response);
final SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
break;
}
final BulkRequestBuilder bulkRequest = client.prepareBulk();
for (final SearchHit hit : hits) {
bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
}
count += hits.length;
final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
if (bulkResponse.hasFailures()) {
throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
}
response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
if (!scrollId.equals(response.getScrollId())) {
deleteScrollContext(scrollId);
}
}
} finally {
deleteScrollContext(scrollId);
}
return count;
}
Example 15
| Source File: ElasticSearchService.java From samantha with MIT License | 5 votes |
|
public BulkResponse bulkDelete(String index, String type, List<String> ids) {
BulkRequestBuilder bulkRequest = client.prepareBulk();
for (String id : ids) {
bulkRequest.add(client.prepareDelete(index, type, id));
}
return bulkRequest.execute().actionGet();
}
Example 16
| Source File: DeIndexOperation.java From usergrid with Apache License 2.0 | 5 votes |
|
@Override
public void doOperation( final Client client, final BulkRequestBuilder bulkRequest ) {
for ( final String index : indexes ) {
final DeleteRequestBuilder builder =
client.prepareDelete( index, IndexingUtils.ES_ENTITY_TYPE, documentId );
bulkRequest.add( builder );
}
}
Example 17
| Source File: ReindexingService.java From elasticsearch-reindexing with Apache License 2.0 | 5 votes |
|
private void sendToLocalCluster(final String scrollId, final SearchHit[] hits) {
// prepare bulk request
final BulkRequestBuilder bulkRequest = client.prepareBulk();
for (final SearchHit hit : hits) {
IndexRequestBuilder builder = client.prepareIndex(toIndex,
toType != null ? toType : hit.getType(), hit.getId())
.setSource(hit.getSource());
Map<String, SearchHitField> fields = hit.getFields();
if (fields != null && fields.containsKey("_parent")) {
SearchHitField parentField = fields.get("_parent");
if (parentField != null) {
String parentId = parentField.getValue();
builder.setParent(parentId);
}
}
bulkRequest.add(builder);
}
// send bulk request, if success response got, searching the next 10 results using scroll_id
// using this listener (inner class) to listen to results
bulkRequest.execute(new ActionListener<BulkResponse>() {
@Override
public void onResponse(final BulkResponse bulkResponse) {
if (bulkResponse.hasFailures()) {
throw new ReindexingException(bulkResponse
.buildFailureMessage());
}
client.prepareSearchScroll(scrollId).setScroll(scroll)
.execute(ReindexingListener.this);
}
@Override
public void onFailure(final Throwable e) {
ReindexingListener.this.onFailure(e);
}
});
}
Example 18
| Source File: SampleIndexTestCase.java From elasticsearch-carrot2 with Apache License 2.0 | 4 votes |
|
@Before
public void createTestIndex() throws Exception {
// Delete any previously indexed content.
client = client();
if (!client.admin().indices().prepareExists(INDEX_TEST).get().isExists()) {
String testTemplate =
"{" +
" \"test\": {" +
" \"properties\": {" +
" \"url\": { \"type\": \"text\" }," +
" \"title\": { \"type\": \"text\" }," +
" \"content\": { \"type\": \"text\" }," +
" \"lang\": { \"type\": \"text\" }," +
" \"rndlang\": { \"type\": \"text\" }" +
" }" +
" }" +
"}";
String emptyTemplate =
"{" +
" \"empty\": {" +
" \"properties\": {" +
" \"url\": { \"type\": \"text\" }," +
" \"title\": { \"type\": \"text\" }," +
" \"content\": { \"type\": \"text\" }," +
" \"lang\": { \"type\": \"text\" }," +
" \"rndlang\": { \"type\": \"text\" }" +
" }" +
" }" +
"}";
CreateIndexResponse response = client.admin().indices()
.prepareCreate(INDEX_TEST)
.addMapping("test", testTemplate, XContentType.JSON)
.get();
Assertions.assertThat(response.isAcknowledged()).isTrue();
response = client.admin().indices()
.prepareCreate(INDEX_EMPTY)
.addMapping("empty", emptyTemplate, XContentType.JSON)
.get();
Assertions.assertThat(response.isAcknowledged()).isTrue();
// Create content at random in the test index.
Random rnd = random();
String[] languages = LanguageComponents.languages().toArray(new String[0]);
Arrays.sort(languages);
BulkRequestBuilder bulk = client.prepareBulk();
for (String[] data : SampleDocumentData.SAMPLE_DATA) {
bulk.add(client.prepareIndex()
.setIndex(INDEX_TEST)
.setType("test")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("url", data[0])
.field("title", data[1])
.field("content", data[2])
.field("lang", "English")
.field("rndlang", languages[rnd.nextInt(languages.length)])
.endObject()));
}
bulk.add(client.prepareIndex()
.setIndex(INDEX_EMPTY)
.setType("empty")
.setSource(XContentFactory.jsonBuilder()
.startObject()
.field("url", "")
.field("title", "")
.field("content", "")
.endObject()));
bulk.execute().actionGet();
flushAndRefresh(INDEX_TEST);
flushAndRefresh(INDEX_EMPTY);
}
ensureGreen(INDEX_TEST);
ensureGreen(INDEX_EMPTY);
InetSocketAddress endpoint = randomFrom(cluster().httpAddresses());
this.restBaseUrl = "http://" + NetworkAddress.format(endpoint);
}
Example 19
| Source File: ESTest.java From canal-1.1.3 with Apache License 2.0 | 4 votes |
|
@Test
public void test04() {
BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
bulkRequestBuilder.add(transportClient.prepareDelete("test", "osm", "2_4"));
commit(bulkRequestBuilder);
}
Example 20
| Source File: ElasticSearchIndexer.java From attic-polygene-java with Apache License 2.0 | 4 votes |
|
private void remove( BulkRequestBuilder bulkBuilder, String identity )
{
bulkBuilder.add( support.client().
prepareDelete( support.index(), support.entitiesType(), identity ) );
}
