org.elasticsearch.action.bulk.BulkResponse Java Examples

/**
 * 批量更新服务
 *
 * @param obj 批量新增请求参数
 * @return <code>true</code>全部更新成功,<code>false</code>部分更新失败.
 * @throws Exception es 执行异常
 */
public SearchBaseResult<Boolean> batchSave(final BatchSaveESObject obj) {
    final BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
    final List<SaveESObject> saveDatas = obj.getSaveDatas();
    for (SaveESObject esObject : saveDatas) {
        bulkRequestBuilder.add(getIndexRequest(esObject));
    }
    if (obj.isRefresh()) {
        bulkRequestBuilder.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
    }
    try {
        SearchLogger.log(bulkRequestBuilder);
        BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
        SearchLogger.log(bulkResponse);
        return SearchBaseResult.success(!bulkResponse.hasFailures(), Boolean.class);
    } catch (Exception ex) {
        SearchLogger.error("batchSave", ex);
        return SearchBaseResult.faild(ESErrorCode.ELASTIC_ERROR_CODE, "esMsg:" + ex.getMessage());
    }
}
 

public Object execute(String options) throws ElasticSearchException {
  try {
  	if(options != null) {
  		if(options.indexOf("refresh=true") > 0) {
		bulkRequestBuilder.setRefreshPolicy("true");
	}
	else if(options.indexOf("refresh=wait_for") > 0){
		bulkRequestBuilder.setRefreshPolicy("wait_for");
	}
	else if(options.indexOf("refresh=false") > 0){
		bulkRequestBuilder.setRefreshPolicy("false");
	}
	else if(options.indexOf("refresh") > 0){
		bulkRequestBuilder.setRefreshPolicy("true");
	}
}
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
      throw new EventDeliveryException(bulkResponse.buildFailureMessage());
    }
    return bulkResponse;
  } finally {
    
  }
}
 

private static void remoteCheck(AbstractClient client, List<Object> builderList) throws ExecutionException, InterruptedException {
  for (Object builder : builderList) {
    BulkRequestBuilder bulkRequestBuilder = null;
    if (builder instanceof IndexRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((IndexRequestBuilder) builder);
    } else if (builder instanceof UpdateRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((UpdateRequestBuilder) builder);
    }  else if (builder instanceof DeleteRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((DeleteRequestBuilder) builder);
    } else {
      fail();
    }
    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().get();
    assertFalse(Arrays.stream(bulkItemResponses.getItems()).anyMatch(BulkItemResponse::isFailed));
  }
}
 

/**
 * 批量删除
 *
 * @param transportClient
 */
private static void batchDelete(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
	DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
	DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");

	bulkRequestBuilder.add(deleteRequestBuilder1);
	bulkRequestBuilder.add(deleteRequestBuilder2);
	bulkRequestBuilder.add(deleteRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}

}
 

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.get();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}
 

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.execute().actionGet();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}
 

@Override
public void insertMany(List<Document> docs)
{
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (Document doc : docs) {
        bulkRequestBuilder.add(new IndexRequest()
                .index(doc.getIndex())
                .type(doc.getType())
                .id(doc.getId())
                .source(doc.getSource()));
    }
    BulkResponse response = bulkRequestBuilder.execute().actionGet();
    if (response.hasFailures()) {
        throw new PrestoException(IO_ERROR, response.buildFailureMessage());
    }
}
 

private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
 

private void bulkDelete(String indexType, JsonNode data) {
    SearchHits hits = elasticSearchService
            .searchHitsByKeys(elasticSearchIndex, indexType, uniqueFields, uniqueFields, data);
    List<String> ids = new ArrayList<>();
    for (SearchHit hit : hits.getHits()) {
        if (hit != null) {
            ids.add(hit.getId());
        }
    }
    if (ids.size() == 0) {
        return;
    }
    BulkResponse resp = elasticSearchService.bulkDelete(elasticSearchIndex, indexType, ids);
    if (resp.hasFailures()) {
        throw new BadRequestException(resp.buildFailureMessage());
    }
}
 

/**
 * Given that streams are processed in bulks, we're making use of
 * the ElasticSearch capability to index bulks of documents. It takes a list
 * of ids and a list of texts.
 *
 * @param tweetIds
 * @param tweets
 */
public void bulkIndex(List<Long> tweetIds, List<String> tweets) {
    BulkRequestBuilder requestBuilder = client.prepareBulk();
    for(int i = 0; i < tweetIds.size(); i++) {
        XContentBuilder builder;
        try {
            builder = jsonBuilder()
                    .startObject()
                    .field("text", tweets.get(i))
                    .field("id", tweetIds.get(i))
                    .endObject();
        } catch (IOException e) {
            continue;
        }
        IndexRequestBuilder request = client.prepareIndex("hackaton", "tweets")
                .setIndex("hackaton")
                .setType("tweets")
                .setSource(builder);
        requestBuilder.add(request);
    }
    BulkResponse bulkResponse = requestBuilder.execute().actionGet();
    int items = bulkResponse.getItems().length;
    System.err.print("indexed [" + items + "] items, with failures? [" + bulkResponse.hasFailures()  + "]");
}
 

@Override
public String bulkDelete(String index, String type, String... ids) {
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (String id : ids) {
			bulkRequest.add(client.prepareDelete(index, type, id));
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 

private void updateDocs(final String type, final List<String> ids, final ObjectNode node) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> {
                bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                    .setRetryOnConflict(NO_OF_CONFLICT_RETRIES)
                    .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON));
            });
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.updatDocs", type, ids, e,
            Metrics.CounterElasticSearchBulkUpdate.getMetricName());
    }
}
 

@Test
public void testIgnoreVersionConficts() throws Exception {
  ArgumentCaptor<BulkProcessor.Listener> listenerCaptor =
          ArgumentCaptor.forClass(BulkProcessor.Listener.class);

  when(BULK_PROCESSOR_FACTORY.getBulkProcessor(eq(CLIENT), listenerCaptor.capture()))
          .thenReturn(processorOne);
  producer.register(SOURCE_ONE);

  BulkResponse response = getRespWithFailedDocument(RestStatus.CONFLICT);

  listenerCaptor.getValue().afterBulk(0, null, response);
  assertEquals(1, metrics.conflicts.getCount());

  producer.flush(SOURCE_ONE);
}
 

@Override
public void bulkDelete(BulkDeleteRequest request) {
    var watch = new StopWatch();
    if (request.ids == null || request.ids.isEmpty()) throw new Error("request.ids must not be empty");

    String index = request.index == null ? this.index : request.index;
    var bulkRequest = new BulkRequest();
    for (String id : request.ids) {
        bulkRequest.add(new org.elasticsearch.action.delete.DeleteRequest(index, id));
    }
    long esTook = 0;
    try {
        BulkResponse response = elasticSearch.client().bulk(bulkRequest, RequestOptions.DEFAULT);
        esTook = response.getTook().nanos();
        if (response.hasFailures()) throw new SearchException(response.buildFailureMessage());
    } catch (IOException e) {
        throw new UncheckedIOException(e);
    } finally {
        long elapsed = watch.elapsed();
        int size = request.ids.size();
        ActionLogContext.track("elasticsearch", elapsed, 0, size);
        logger.debug("bulkDelete, index={}, ids={}, size={}, esTook={}, elapsed={}", index, request.ids, size, esTook, elapsed);
        checkSlowOperation(elapsed);
    }
}
 

@Test
public void testRunError() throws IOException {
    AnomalyModel anomalyModel = AnomalyModel.newBuilder()
            .key("key")
            .value(100)
            .level("NORMAL")
            .uuid("test")
            .timestamp("date")
            .anomalyThresholds(null)
            .tags(null)
            .build();
    List<AnomalyModel> anomalyModels = new ArrayList<>();
    anomalyModels.add(anomalyModel);
    BulkResponse bulkResponse = buildBulkResponseError();

    when(client.bulk(any(BulkRequest.class), any(RequestOptions.class))).thenReturn(bulkResponse);
    when(client.close()).thenReturn(true);
    ElasticSearchBulkService elasticSearchBulkService = new ElasticSearchBulkService(anomalyModels);
    elasticSearchBulkService.setElasticSearchClient(client);
    elasticSearchBulkService.run();
    verify(elasticSearchBulkService.getElasticSearchClient(), times(1))
            .bulk(any(BulkRequest.class), any(RequestOptions.class));
    verify(bulkResponse,times(1)).buildFailureMessage();
}
 

@Override
public List<String> deleteAllIds(final Collection<String> ids) {
  if (ids.isEmpty()) {
    return ImmutableList.of();
  }

  final BulkRequestBuilder bulk = client
    .prepareBulk()
    .setRefreshPolicy(policy.get());

  for (final String id : ids) {
    bulk.add(client.prepareDelete(index, type, id));
  }

  final BulkResponse response = bulk.execute().actionGet();

  final ImmutableList.Builder<String> builder = ImmutableList.builder();
  for (final BulkItemResponse item : response.getItems()) {
    builder.add(item.getId());
  }
  return builder.build();
}
 

private static void BulkIndex(Client client) throws IOException {

        BulkRequestBuilder requestBuilder = client.prepareBulk();

        requestBuilder.add(client.prepareIndex("twitter","tweet","4")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","niekaijie")
                        .field("school","beiyou")
                        .field("address","haidianqu")
                        .endObject())
        );
        requestBuilder.add(client.prepareIndex("twitter","tweet","3")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","林志颖aa")
                        .field("school","台湾大学")
                        .field("address","台湾")
                        .endObject())
        );
        BulkResponse response = requestBuilder.get();


    }
 

private void assertGetSpan(AbstractTracingSpan getSpan, Object ret) {
    assertThat(getSpan instanceof LocalSpan, is(true));

    LocalSpan span = (LocalSpan) getSpan;
    assertThat(span.getOperationName(), is("Elasticsearch/actionGet"));
    assertThat(SpanHelper.getComponentId(span), is(TRANSPORT_CLIENT.getId()));

    List<TagValuePair> tags = SpanHelper.getTags(span);
    assertThat(tags.size(), is(4));
    if (ret instanceof SearchResponse) {
        assertThat(tags.get(0).getValue(), is("Elasticsearch"));
        assertThat(tags.get(1).getValue(), is("2020"));
        assertThat(tags.get(2).getValue(), is("309"));
    } else if (ret instanceof BulkResponse) {
        assertThat(tags.get(0).getValue(), is("Elasticsearch"));
        assertThat(tags.get(1).getValue(), is("2020"));
        assertThat(tags.get(2).getValue(), is("1416"));
    }

}
 

protected void bulkUpdateIndexes( ) {

        List<Tuple> inputs = new ArrayList<>(queue.size());
        queue.drainTo(inputs);
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (Tuple input : inputs) {
            Document<T> doc = mapper.map(input);
            IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource((String)doc.getSource());

            if(doc.getParentId() != null) {
                request.setParent(doc.getParentId());
            }
            bulkRequest.add(request);
        }

        try {
            if (bulkRequest.numberOfActions() > 0) {
                BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
                if (bulkItemResponses.hasFailures()) {
                    BulkItemResponse[] items = bulkItemResponses.getItems();
                    for (int i = 0; i < items.length; i++) {
                        ackOrFail(items[i], inputs.get(i));
                    }
                } else {
                    ackAll(inputs);
                }
            }
        } catch (ElasticsearchException e) {
            LOGGER.error("Unable to process bulk request, " + inputs.size() + " tuples are in failure", e);
            outputCollector.reportError(e.getRootCause());
            failAll(inputs);
        }
    }
 

/**
 * @return boolean
 * @Author pancm
 * @Description 批量新增/更新数据
 * @Date 2019/3/21
 * @Param [mapList:存储参数, index:索引库名, type:索引库类型,key:存储的主键，为空表示使用ES主键]
 **/
public static boolean saveBulk(List<Map<String, Object>> mapList, String index, String type, String key) throws IOException {

    if (mapList == null || mapList.size() == 0) {
        return true;
    }
    if (index == null || index.trim().length() == 0 || type == null || type.trim().length() == 0) {
        return false;
    }
    try {
        BulkRequest request = new BulkRequest();
        mapList.forEach(map -> {
            if (key != null) {
                String id = map.get(key) + "";
                if (id == null || id.trim().length() == 0) {
                    request.add(new IndexRequest(index, type).source(map, XContentType.JSON));
                } else {
                    request.add(new IndexRequest(index, type, id).source(map, XContentType.JSON));
                }
            } else {
                request.add(new IndexRequest(index, type).source(map, XContentType.JSON));
            }
        });

        BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
        //说明至少有一个失败了，这里就直接返回false
        if (bulkResponse.hasFailures()) {
            return false;
        }

        return true;
    } finally {
        if (isAutoClose) {
            close();
        }
    }
}
 

@Test
public void testAdd() throws IOException {

    final Map<String, Object> doc = new HashMap<>();
    doc.put("dynamic_string_title", "The last ascent of man");
    doc.put(FieldUtil.ID, "AA-2X3451");
    doc.put(FieldUtil.TYPE, "TestDoc");
    final BulkResponse indexResult = client.add(doc);
    assertNotNull(indexResult);
    assertEquals("CREATED", indexResult.getItems()[0].status().name());
}
 

public boolean bulkUpdate(final Consumer<BulkRequestBuilder> consumer) {
    final BulkRequestBuilder builder = ComponentUtil.getFessEsClient().prepareBulk();
    consumer.accept(builder);
    try {
        final BulkResponse response = builder.execute().get();
        if (response.hasFailures()) {
            throw new FessEsClientException(response.buildFailureMessage());
        } else {
            return true;
        }
    } catch (InterruptedException | ExecutionException e) {
        throw new FessEsClientException("Failed to update bulk data.", e);
    }
}
 

@Override
public void handle(BulkResponse response) {

    if( response.hasFailures() ) {
        LOGGER.error("BulkResponse has failures : {}", response.buildFailureMessage());
    }
}
 

@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item = mock(BulkItemResponse.class);
    when(item.getItemId()).thenReturn(1);
    when(item.isFailed()).thenReturn(true);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item});
    TransportAddress remoteAddress = mock(TransportAddress.class);
    when(remoteAddress.getAddress()).thenReturn(address.toString());
    when(response.remoteAddress()).thenReturn(remoteAddress);
    return response;
}
 

public void execute(BulkRequest bulkRequest, long executionId) {
    boolean afterCalled = false;
    try {
        listener.beforeBulk(executionId, bulkRequest);
        BulkResponse bulkResponse = client.execute(BulkAction.INSTANCE, bulkRequest).actionGet();
        afterCalled = true;
        listener.afterBulk(executionId, bulkRequest, bulkResponse);
    } catch (Throwable t) {
        if (!afterCalled) {
            listener.afterBulk(executionId, bulkRequest, t);
        }
    }
}
 

private static void reindexData(IndicesAdminClient indicesAdminClient) {
    //查询原来的所有数据,TimeValue是需要保存的时长
    SearchResponse searchResponse = client.prepareSearch(ALIX_NAME).setTypes(INDEX_TYPE).setQuery(QueryBuilders.matchAllQuery()).
            setSearchType(SearchType.SCAN).setScroll(new TimeValue(20000))
            .setSize(100).execute().actionGet();
    //当前id
         String scrollId = searchResponse.getScrollId();
        while(StringUtils.isNotEmpty(scrollId)) {
            BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
            SearchResponse scrollSearchResponse = client.prepareSearchScroll(scrollId).setScroll(new TimeValue(20000)).execute().actionGet();
            SearchHits hits = scrollSearchResponse.getHits();
            if(hits.getHits().length>0) {
                for (SearchHit searchHitFields : hits.getHits()) {
                    IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v2, INDEX_TYPE).setId(searchHitFields.getId()).setSource(searchHitFields.getSource()).setOpType(IndexRequest.OpType.INDEX);
                    bulkRequestBuilder.add(indexRequestBuilder);
                }
                BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
                if (bulkResponse.hasFailures()) {
                    log.error("reindex失败 : {}", bulkResponse.buildFailureMessage());
                } else {
                    log.info("reindex {}条成功:", hits.getHits().length);
                }
            }else{
                break;
            }
        }
}
 

@Override
public void storeSpan(String tenantId, List<Span> spans, Function<Span, String> spanIdSupplier)
        throws StoreException {

    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    for (Span span : spans) {
        String json;
        try {
            json = serialize(span);
        } catch(IOException ex){
            log.errorf("Failed to serialize span %s", span);
            throw new StoreException(ex);
        }

        log.tracef("Storing span: %s", json);
        // modified id is used in index
        final String modifiedId = spanIdSupplier.apply(span);

        bulkRequestBuilder.add(client.getClient()
                .prepareIndex(client.getIndex(tenantId), SPAN_TYPE, modifiedId)
                .setSource(json));
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {
        log.tracef("Failed to store spans to elasticsearch: %s", bulkItemResponses.buildFailureMessage());
        throw new StoreException(bulkItemResponses.buildFailureMessage());
    }

    log.trace("Success storing spans to elasticsearch");
}
 

@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        log.warn("Bulk [{" + executionId + "}] executed with failures");
    } else if (log.isDebugEnabled()) {
        log.debug("Bulk [{" + executionId + "}] completed in {" + response.getTook().getMillis() +
                "} milliseconds");
    }
}
 

private void flushBulk() {
	BulkResponse bulkResponse = bulkRequest.execute().actionGet(); // flush
	if (bulkResponse.hasFailures()) { // log failures
		for (BulkItemResponse r : bulkResponse.getItems()) {
			LOG.error(r.getFailureMessage());
			Failure failure = r.getFailure();
			// e.g. when ES server is overloaded
			throw new ElasticsearchException(failure.toString());
		}
	}
}
 

/**
 * 批量更新网页
 *
 * @param webpageList 网页列表
 * @return
 */
public boolean update(List<Webpage> webpageList) throws ExecutionException, InterruptedException {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (Webpage webpage : webpageList) {
        UpdateRequest updateRequest = new UpdateRequest();
        updateRequest.index(INDEX_NAME);
        updateRequest.type(TYPE_NAME);
        updateRequest.id(webpage.getId());
        updateRequest.doc(gson.toJson(webpage));
        bulkRequest.add(updateRequest);
    }
    BulkResponse bulkResponse = bulkRequest.get();
    return bulkResponse.hasFailures();
}