Java Code Examples for org.elasticsearch.action.bulk.BulkItemResponse#getFailure()

The following examples show how to use org.elasticsearch.action.bulk.BulkItemResponse#getFailure() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: Output.java    From data-generator with Apache License 2.0 6 votes vote down vote up
private static void writeBatchToES(String index, String type, List<Map<String, Object>> list) throws Exception{
    if(list.isEmpty()){
        return;
    }
    BulkRequest request = new BulkRequest();
    for(Map<String, Object> data : list) {
        String id = data.get("id").toString();
        request.add(
                new IndexRequest(index, type, id)
                        .source(data));

    }
    BulkResponse bulkResponse = CLIENT.bulk(request);
    if (bulkResponse.hasFailures()) {
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            if (bulkItemResponse.isFailed()) {
                BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
                LOGGER.error("ES索引失败: {}", failure.getMessage());
            }
        }
    }
}
 
Example 2
Source File: BulkUpdateService.java    From vertexium with Apache License 2.0 6 votes vote down vote up
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
 
Example 3
Source File: BulkUpdateService.java    From vertexium with Apache License 2.0 6 votes vote down vote up
private void handleFailure(BulkItem<?> bulkItem, BulkItemResponse bulkItemResponse) {
    BulkItemResponse.Failure failure = bulkItemResponse.getFailure();
    bulkItem.incrementFailCount();
    if (bulkItem.getFailCount() >= maxFailCount) {
        complete(bulkItem, new BulkVertexiumException("fail count exceeded the max number of failures", failure));
    } else {
        AtomicBoolean retry = new AtomicBoolean(false);
        try {
            searchIndex.handleBulkFailure(bulkItem, bulkItemResponse, retry);
        } catch (Exception ex) {
            complete(bulkItem, ex);
            return;
        }
        if (retry.get()) {
            incomingItems.addAll(bulkItem.getItems());
        } else {
            complete(bulkItem, null);
        }
    }
}
 
Example 4
Source File: ElasticIndexer.java    From bluima with Apache License 2.0 5 votes vote down vote up
private void flushBulk() {
	BulkResponse bulkResponse = bulkRequest.execute().actionGet(); // flush
	if (bulkResponse.hasFailures()) { // log failures
		for (BulkItemResponse r : bulkResponse.getItems()) {
			LOG.error(r.getFailureMessage());
			Failure failure = r.getFailure();
			// e.g. when ES server is overloaded
			throw new ElasticsearchException(failure.toString());
		}
	}
}
 
Example 5
Source File: FessEsClient.java    From fess with Apache License 2.0 5 votes vote down vote up
public void addAll(final String index, final List<Map<String, Object>> docList,
        final BiConsumer<Map<String, Object>, IndexRequestBuilder> options) {
    final FessConfig fessConfig = ComponentUtil.getFessConfig();
    final BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (final Map<String, Object> doc : docList) {
        final Object id = doc.remove(fessConfig.getIndexFieldId());
        final IndexRequestBuilder builder = client.prepareIndex().setIndex(index).setId(id.toString()).setSource(new DocMap(doc));
        options.accept(doc, builder);
        bulkRequestBuilder.add(builder);
    }
    final BulkResponse response = bulkRequestBuilder.execute().actionGet(ComponentUtil.getFessConfig().getIndexBulkTimeout());
    if (response.hasFailures()) {
        if (logger.isDebugEnabled()) {
            final List<DocWriteRequest<?>> requests = bulkRequestBuilder.request().requests();
            final BulkItemResponse[] items = response.getItems();
            if (requests.size() == items.length) {
                for (int i = 0; i < requests.size(); i++) {
                    final BulkItemResponse resp = items[i];
                    if (resp.isFailed() && resp.getFailure() != null) {
                        final DocWriteRequest<?> req = requests.get(i);
                        final Failure failure = resp.getFailure();
                        logger.debug("Failed Request: {}\n=>{}", req, failure.getMessage());
                    }
                }
            }
        }
        throw new FessEsClientException(response.buildFailureMessage());
    }
}
 
Example 6
Source File: StatusUpdaterBolt.java    From storm-crawler with Apache License 2.0 4 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    LOG.debug("afterBulk [{}] with {} responses", executionId,
            request.numberOfActions());
    long msec = response.getTook().getMillis();
    eventCounter.scope("bulks_received").incrBy(1);
    eventCounter.scope("bulk_msec").incrBy(msec);
    Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
    int itemcount = 0;
    int acked = 0;
    int failurecount = 0;

    synchronized (waitAck) {
        while (bulkitemiterator.hasNext()) {
            BulkItemResponse bir = bulkitemiterator.next();
            itemcount++;
            String id = bir.getId();
            BulkItemResponse.Failure f = bir.getFailure();
            boolean failed = false;
            if (f != null) {
                // already discovered
                if (f.getStatus().equals(RestStatus.CONFLICT)) {
                    eventCounter.scope("doc_conflicts").incrBy(1);
                    LOG.debug("Doc conflict ID {}", id);
                } else {
                    LOG.error("Update ID {}, failure: {}", id, f);
                    failed = true;
                }
            }
            List<Tuple> xx = waitAck.getIfPresent(id);
            if (xx != null) {
                LOG.debug("Acked {} tuple(s) for ID {}", xx.size(), id);
                for (Tuple x : xx) {
                    if (!failed) {
                        String url = x.getStringByField("url");
                        acked++;
                        // ack and put in cache
                        LOG.debug("Acked {} with ID {}", url, id);
                        super.ack(x, url);
                    } else {
                        failurecount++;
                        _collector.fail(x);
                    }
                }
                waitAck.invalidate(id);
            } else {
                LOG.warn("Could not find unacked tuple for {}", id);
            }
        }

        LOG.info(
                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
                executionId, itemcount, waitAck.size(), acked, failurecount);
        if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
            for (String kinaw : waitAck.asMap().keySet()) {
                LOG.debug(
                        "Still in wait ack after bulk response [{}] => {}",
                        executionId, kinaw);
            }
        }
    }
}
 
Example 7
Source File: IndexerBolt.java    From storm-crawler with Apache License 2.0 4 votes vote down vote up
@Override
public void afterBulk(long executionId, BulkRequest request,
        BulkResponse response) {
    long msec = response.getTook().getMillis();
    eventCounter.scope("bulks_received").incrBy(1);
    eventCounter.scope("bulk_msec").incrBy(msec);
    Iterator<BulkItemResponse> bulkitemiterator = response.iterator();
    int itemcount = 0;
    int acked = 0;
    int failurecount = 0;

    synchronized (waitAck) {
        while (bulkitemiterator.hasNext()) {
            BulkItemResponse bir = bulkitemiterator.next();
            itemcount++;
            String id = bir.getId();
            BulkItemResponse.Failure f = bir.getFailure();
            boolean failed = false;
            if (f != null) {
                if (f.getStatus().equals(RestStatus.CONFLICT)) {
                    eventCounter.scope("doc_conflicts").incrBy(1);
                } else {
                    failed = true;
                }
            }
            Tuple t = waitAck.getIfPresent(id);
            if (t == null) {
                LOG.warn("Could not find unacked tuple for {}", id);
                continue;
            }

            LOG.debug("Acked  tuple for ID {}", id);
            String u = (String) t.getValueByField("url");

            Metadata metadata = (Metadata) t.getValueByField("metadata");

            if (!failed) {
                acked++;
                _collector.emit(StatusStreamName, t, new Values(u,
                        metadata, Status.FETCHED));
                _collector.ack(t);
            } else {
                failurecount++;
                LOG.error("update ID {}, URL {}, failure: {}", id, u, f);
                // there is something wrong with the content we should treat
                // it as an ERROR
                if (f.getStatus().equals(RestStatus.BAD_REQUEST)) {
                    metadata.setValue(Constants.STATUS_ERROR_SOURCE,
                            "ES indexing");
                    metadata.setValue(Constants.STATUS_ERROR_MESSAGE,
                            "invalid content");
                    _collector.emit(StatusStreamName, t, new Values(u,
                            metadata, Status.ERROR));
                    _collector.ack(t);
                }
                // otherwise just fail it
                else {
                    _collector.fail(t);
                }
            }
            waitAck.invalidate(id);
        }

        LOG.info(
                "Bulk response [{}] : items {}, waitAck {}, acked {}, failed {}",
                executionId, itemcount, waitAck.size(), acked, failurecount);

        if (waitAck.size() > 0 && LOG.isDebugEnabled()) {
            for (String kinaw : waitAck.asMap().keySet()) {
                LOG.debug(
                        "Still in wait ack after bulk response [{}] => {}",
                        executionId, kinaw);
            }
        }
    }
}