Java Code Examples for org.elasticsearch.action.bulk.BulkItemResponse

The following examples show how to use org.elasticsearch.action.bulk.BulkItemResponse. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: canal-1.1.3   Source File: ESTemplate.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 提交批次
 */
public void commit() {
    if (getBulk().numberOfActions() > 0) {
        BulkResponse response = getBulk().execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    logger.error(itemResponse.getFailureMessage());
                } else {
                    throw new RuntimeException("ES sync commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 
Example 2
Source Project: canal-1.1.3   Source File: ESTest.java    License: Apache License 2.0 6 votes vote down vote up
private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 
Example 3
Source Project: garmadon   Source File: ElasticSearchListener.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
    if (response.hasFailures()) {
        LOGGER.error("Bulk[{}] executed with failures", executionId);
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                LOGGER.error("Failed on {} due to {}", item.getId(), item.getFailureMessage());
                numberOfEventInError.inc();
            }
        }
    } else {
        LOGGER.info("Successfully completed Bulk[{}] in {} ms", executionId, response.getTook().getMillis());
        latencyIndexingEvents.observe(response.getTook().getMillis());
    }
    CommittableOffset<String, byte[]> lastOffset = (CommittableOffset<String, byte[]>) request.payloads().get(request.payloads().size() - 1);
    lastOffset
            .commitAsync()
            .whenComplete((topicPartitionOffset, exception) -> {
                if (exception != null) {
                    LOGGER.warn("Could not commit kafka offset {}|{}", lastOffset.getOffset(), lastOffset.getPartition());
                    numberOfOffsetCommitError.inc();
                } else {
                    LOGGER.info("Committed kafka offset {}|{}", topicPartitionOffset.getOffset(), topicPartitionOffset.getPartition());
                }
            });
}
 
Example 4
Source Project: nifi   Source File: TestPutElasticsearch.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
 
Example 5
Source Project: localization_nifi   Source File: TestPutElasticsearch.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public BulkResponse get() throws InterruptedException, ExecutionException {
    BulkResponse response = mock(BulkResponse.class);
    when(response.hasFailures()).thenReturn(responseHasFailures);
    BulkItemResponse item1 = mock(BulkItemResponse.class);
    BulkItemResponse item2 = mock(BulkItemResponse.class);
    when(item1.getItemId()).thenReturn(1);
    when(item1.isFailed()).thenReturn(true);
    BulkItemResponse.Failure failure = mock(BulkItemResponse.Failure.class);
    when(failure.getMessage()).thenReturn("Bad message");
    when(item1.getFailure()).thenReturn(failure);
    when(item2.getItemId()).thenReturn(2);
    when(item2.isFailed()).thenReturn(false);
    when(response.getItems()).thenReturn(new BulkItemResponse[]{item1, item2});
    return response;
}
 
Example 6
Source Project: data-generator   Source File: Output.java    License: Apache License 2.0 6 votes vote down vote up
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
 
Example 7
private static void remoteCheck(AbstractClient client, List<Object> builderList) throws ExecutionException, InterruptedException {
  for (Object builder : builderList) {
    BulkRequestBuilder bulkRequestBuilder = null;
    if (builder instanceof IndexRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((IndexRequestBuilder) builder);
    } else if (builder instanceof UpdateRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((UpdateRequestBuilder) builder);
    }  else if (builder instanceof DeleteRequestBuilder) {
      bulkRequestBuilder = client.prepareBulk().add((DeleteRequestBuilder) builder);
    } else {
      fail();
    }
    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().get();
    assertFalse(Arrays.stream(bulkItemResponses.getItems()).anyMatch(BulkItemResponse::isFailed));
  }
}
 
Example 8
/**
 * 批量删除
 *
 * @param transportClient
 */
private static void batchDelete(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
	DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
	DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");

	bulkRequestBuilder.add(deleteRequestBuilder1);
	bulkRequestBuilder.add(deleteRequestBuilder2);
	bulkRequestBuilder.add(deleteRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}

}
 
Example 9
Source Project: metacat   Source File: ElasticSearchUtilImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> {
                bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                    .setRetryOnConflict(NO_OF_CONFLICT_RETRIES)
                    .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON));
            });
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.updatDocs", type, ids, e,
            Metrics.CounterElasticSearchBulkUpdate.getMetricName());
    }
}
 
Example 10
Source Project: metacat   Source File: ElasticSearchUtilImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Permanently delete index documents.
 *
 * @param type index type
 * @param ids  entity ids
 */
private void hardDeleteDoc(final String type, final List<String> ids) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> bulkRequest.add(client.prepareDelete(esIndex, type, id)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            log.info("Deleting metadata of type {} with count {}", type, ids.size());
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.bulkDelete.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.bulkDelete", type, ids, e,
            Metrics.CounterElasticSearchBulkDelete.getMetricName());
    }
}
 
Example 11
@Override
public <T extends Entity> boolean bulkUpdate(String indexName, List<? extends Entity> entities) throws IOException {
    BulkRequest bulkRequest = new BulkRequest();
    entities.stream().map(e -> createUpdateRequest(indexName, getType(e), e.getId(), getJson(e), getParent(e), getRoot(e))).
            forEach(bulkRequest::add);
    bulkRequest.setRefreshPolicy(esCfg.refreshPolicy);

    BulkResponse bulkResponse = client.bulk(bulkRequest);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse resp : bulkResponse.getItems()) {
            if (resp.isFailed()) {
                LOGGER.error("bulk update failed : {}", resp.getFailureMessage());
            }
        }
        return false;
    }
    return true;
}
 
Example 12
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    bulksInProgress.decrementAndGet();
    if (response.hasFailures()) {
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                counts.failures++;
            } else {
                counts.successes++;
            }
        }
    } else {
        counts.successes += response.getItems().length;
    }
    checkRelease();
}
 
Example 13
Source Project: vertexium   Source File: BulkUpdateService.java    License: Apache License 2.0 6 votes vote down vote up
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
 
Example 14
@Override
public void handleBulkFailure(
    Graph graph,
    Elasticsearch7SearchIndex elasticsearch7SearchIndex,
    BulkItem<?> bulkItem,
    BulkItemResponse bulkItemResponse,
    AtomicBoolean retry
) {
    numberOfTimesCalled.incrementAndGet();
    LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure());
    if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
        LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch7SearchIndex, bulkItem, authorizations);
    } else {
        retry.set(true);
    }
}
 
Example 15
@Override
public void handleBulkFailure(
    Graph graph,
    Elasticsearch5SearchIndex elasticsearch5SearchIndex,
    BulkItem<?> bulkItem,
    BulkItemResponse bulkItemResponse,
    AtomicBoolean retry
) {
    numberOfTimesCalled.incrementAndGet();
    LOGGER.warn("bulk failure on item %s: %s", bulkItem, bulkItemResponse == null ? null : bulkItemResponse.getFailure());
    if (bulkItemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
        LoadAndAddDocumentMissingHelper.handleDocumentMissingException(graph, elasticsearch5SearchIndex, bulkItem, authorizations);
    } else {
        retry.set(true);
    }
}
 
Example 16
Source Project: elasticsearch-river-kafka   Source File: KafkaRiver.java    License: Apache License 2.0 6 votes vote down vote up
void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}
 
Example 17
Source Project: Flink-CEPplus   Source File: Elasticsearch1ApiCallBridge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return new RuntimeException(bulkItemResponse.getFailureMessage());
	}
}
 
Example 18
Source Project: Flink-CEPplus   Source File: Elasticsearch2ApiCallBridge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}
 
Example 19
Source Project: Flink-CEPplus   Source File: Elasticsearch5ApiCallBridge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: ElasticsearchSinkBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
	if (response.hasFailures()) {
		BulkItemResponse itemResponse;
		Throwable failure;
		RestStatus restStatus;

		try {
			for (int i = 0; i < response.getItems().length; i++) {
				itemResponse = response.getItems()[i];
				failure = callBridge.extractFailureCauseFromBulkItemResponse(itemResponse);
				if (failure != null) {
					LOG.error("Failed Elasticsearch item request: {}", itemResponse.getFailureMessage(), failure);

					restStatus = itemResponse.getFailure().getStatus();
					if (restStatus == null) {
						failureHandler.onFailure(request.requests().get(i), failure, -1, failureRequestIndexer);
					} else {
						failureHandler.onFailure(request.requests().get(i), failure, restStatus.getStatus(), failureRequestIndexer);
					}
				}
			}
		} catch (Throwable t) {
			// fail the sink and skip the rest of the items
			// if the failure handler decides to throw an exception
			failureThrowable.compareAndSet(null, t);
		}
	}

	if (flushOnCheckpoint) {
		numPendingRequests.getAndAdd(-request.numberOfActions());
	}
}
 
Example 21
@SneakyThrows
protected void checkResponse(BulkResponse response) {
    if (response.hasFailures()) {
        for (BulkItemResponse item : response.getItems()) {
            if (item.isFailed()) {
                throw item.getFailure().getCause();
            }
        }
    }
}
 
Example 22
Source Project: nutch-htmlunit   Source File: ElasticIndexWriter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void commit() throws IOException {
  if (execute != null) {
    // wait for previous to finish
    long beforeWait = System.currentTimeMillis();
    BulkResponse actionGet = execute.actionGet();
    if (actionGet.hasFailures()) {
      for (BulkItemResponse item : actionGet) {
        if (item.isFailed()) {
          throw new RuntimeException("First failure in bulk: "
                  + item.getFailureMessage());
        }
      }
    }
    long msWaited = System.currentTimeMillis() - beforeWait;
    LOG.info("Previous took in ms " + actionGet.getTookInMillis()
            + ", including wait " + msWaited);
    execute = null;
  }
  if (bulk != null) {
    if (bulkDocs > 0) {
      // start a flush, note that this is an asynchronous call
      execute = bulk.execute();
    }
    bulk = null;
  }
  if (createNewBulk) {
    // Prepare a new bulk request
    bulk = client.prepareBulk();
    bulkDocs = 0;
    bulkLength = 0;
  }
}
 
Example 23
Source Project: flink   Source File: Elasticsearch5ApiCallBridge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}
 
Example 24
Source Project: flink   Source File: Elasticsearch6ApiCallBridge.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (!bulkItemResponse.isFailed()) {
		return null;
	} else {
		return bulkItemResponse.getFailure().getCause();
	}
}
 
Example 25
Source Project: flink   Source File: ElasticsearchSinkBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
	if (response.hasFailures()) {
		BulkItemResponse itemResponse;
		Throwable failure;
		RestStatus restStatus;

		try {
			for (int i = 0; i < response.getItems().length; i++) {
				itemResponse = response.getItems()[i];
				failure = callBridge.extractFailureCauseFromBulkItemResponse(itemResponse);
				if (failure != null) {
					LOG.error("Failed Elasticsearch item request: {}", itemResponse.getFailureMessage(), failure);

					restStatus = itemResponse.getFailure().getStatus();
					if (restStatus == null) {
						failureHandler.onFailure(request.requests().get(i), failure, -1, failureRequestIndexer);
					} else {
						failureHandler.onFailure(request.requests().get(i), failure, restStatus.getStatus(), failureRequestIndexer);
					}
				}
			}
		} catch (Throwable t) {
			// fail the sink and skip the rest of the items
			// if the failure handler decides to throw an exception
			failureThrowable.compareAndSet(null, t);
		}
	}

	if (flushOnCheckpoint) {
		numPendingRequests.getAndAdd(-request.numberOfActions());
	}
}
 
Example 26
Source Project: flink   Source File: ElasticsearchSinkBaseTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public Throwable extractFailureCauseFromBulkItemResponse(BulkItemResponse bulkItemResponse) {
	if (bulkItemResponse.isFailed()) {
		return new Exception(bulkItemResponse.getFailure().getMessage());
	} else {
		return null;
	}
}
 
Example 27
Source Project: canal-1.1.3   Source File: ESEtlService.java    License: Apache License 2.0 5 votes vote down vote up
private void processFailBulkResponse(BulkResponse bulkResponse) {
    for (BulkItemResponse response : bulkResponse.getItems()) {
        if (!response.isFailed()) {
            continue;
        }

        if (response.getFailure().getStatus() == RestStatus.NOT_FOUND) {
            logger.warn(response.getFailureMessage());
        } else {
            logger.error("全量导入数据有误 {}", response.getFailureMessage());
            throw new RuntimeException("全量数据 etl 异常: " + response.getFailureMessage());
        }
    }
}
 
Example 28
Source Project: SkaETL   Source File: ErrorToElasticsearchProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void parseResultErrors(BulkRequest request, BulkResponse bulkItemResponses) {
    for (BulkItemResponse bir : bulkItemResponses) {
        MDC.put("item_error", bir.getFailureMessage());
        log.info("EsError" + bir.getFailureMessage());
        MDC.remove("item_error");
        //TODO ...
    }
}
 
Example 29
Source Project: SkaETL   Source File: AbstractElasticsearchProcessor.java    License: Apache License 2.0 5 votes vote down vote up
protected void parseResultErrors(BulkRequest request, BulkResponse bulkItemResponses) {
    for (BulkItemResponse bir : bulkItemResponses) {
        DocWriteRequest docWriteRequest = request.requests().get(bir.getItemId());
        if (bir.isFailed()) {
            if (isRetryable(bir)) {
                routeToNextTopic(bir, toRawMessage(docWriteRequest), false);
            } else {
                routeToNextTopic(bir, toRawMessage(docWriteRequest), true);
            }
        }
    }
}
 
Example 30
Source Project: elasticsearch-jest-example   Source File: TransportClient.java    License: MIT License 5 votes vote down vote up
/**
 * 删除查询到的文档
 * @param index
 * @param name
 * @param value
 */
private static void scrollSearchDelete(String index,String name,String value){
	Client client = createTransportClient();
	QueryBuilder qb = termQuery(name, value);
	SearchResponse scrollResp = client.prepareSearch(index)
			.setSearchType(SearchType.SCAN)
			.setScroll(new TimeValue(60000))
			.setQuery(qb)
			.setSize(100).execute().actionGet(); //100 hits per shard will be returned for each scroll

	BulkRequestBuilder bulkRequest = client.prepareBulk();

	while (true) {
		for (SearchHit hit : scrollResp.getHits().getHits()) {
			bulkRequest.add(client.prepareDelete(hit.getIndex(),hit.getType(),hit.getId()));
		}
		scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
		if (scrollResp.getHits().getHits().length == 0) {
			break;
		}
	}
	BulkResponse bulkResponse = bulkRequest.execute().actionGet();
	if (bulkResponse.hasFailures()) {
		BulkItemResponse[] bulkItemResponse = bulkResponse.getItems();
		for (int i = 0; i <bulkItemResponse.length ; i++) {
			System.out.println(bulkItemResponse[i].getItemId()+":"+bulkItemResponse[i].getIndex()+":"+bulkItemResponse[i].getFailureMessage());
		}
	}
}