Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder#numberOfActions()

The following examples show how to use org.elasticsearch.action.bulk.BulkRequestBuilder#numberOfActions() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: CassandraRiver.java From cassandra-river with Apache License 2.0

6 votes

@Override
public void run() {
	logger.info("Starting thread with {} keys", this.keys.rowColumnMap.size());
              if (closed) {
                  return;
              }
              
              BulkRequestBuilder bulk = client.prepareBulk();
              for(String key : this.keys.rowColumnMap.keySet()){
              	
  				try {
  					String id = UUID.nameUUIDFromBytes(key.getBytes()).toString();
  					bulk.add(indexRequest(this.indexName).type(this.typeName)
					    							 .id(id)
					    							 .source(this.keys.rowColumnMap.get(key)));
  				} catch (Exception e) {
  					logger.error("failed to entry to bulk indexing");
  				}
  				
              	if(bulk.numberOfActions() >= this.batchSize){
              		saveToEs(bulk);
  					bulk = client.prepareBulk();
              	}
              }
}

Example 2

Source File: ESIndexState.java From storm-trident-elasticsearch with Apache License 2.0

6 votes

public void bulkUpdateIndices(List<TridentTuple> inputs, TridentTupleMapper<Document<T>> mapper, BulkResponseHandler handler) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for (TridentTuple input : inputs) {
        Document<T> doc = mapper.map(input);
        byte[] source = serializeSourceOrFail(doc);
        IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource(source);

        if(doc.getParentId() != null) {
            request.setParent(doc.getParentId());
        }
        bulkRequest.add(request);
    }

    if( bulkRequest.numberOfActions() > 0) {
        try {
            handler.handle(bulkRequest.execute().actionGet());
        } catch(ElasticsearchException e) {
            LOGGER.error("error while executing bulk request to elasticsearch");
            throw new FailedException("Failed to store data into elasticsearch", e);
        }
    }
}

Example 3

Source File: ESTest.java From canal-1.1.3 with Apache License 2.0

6 votes

private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}

Example 4

Source File: ESUpdateState.java From sql4es with Apache License 2.0

6 votes

/**
 * Executes the list with requests as a bulk with maximum number of requests per bulk
 * @param requests
 * @param maxRequestsPerBulk
 * @return
 * @throws SQLException
 */
private int execute(List<?> requests, int maxRequestsPerBulk) throws SQLException{
	int result = 0;
	BulkRequestBuilder bulkReq = client.prepareBulk();
	for(Object req : requests){
		if(req instanceof IndexRequest)	bulkReq.add((IndexRequest)req);
		else if(req instanceof UpdateRequest) bulkReq.add((UpdateRequest)req);
		else if(req instanceof DeleteRequest) bulkReq.add((DeleteRequest)req);
		else if(req instanceof IndexRequestBuilder) bulkReq.add((IndexRequestBuilder)req);
		else if(req instanceof UpdateRequestBuilder) bulkReq.add((UpdateRequestBuilder)req);
		else if(req instanceof DeleteRequestBuilder) bulkReq.add((DeleteRequestBuilder)req);
		else throw new SQLException("Type "+req.getClass()+" cannot be added to a bulk request");
		
		if(bulkReq.numberOfActions() > maxRequestsPerBulk){
			result += bulkReq.get().getItems().length;
			bulkReq = client.prepareBulk();
		}
	}
	
	if(bulkReq.numberOfActions() > 0){
		result += bulkReq.get().getItems().length;
	}
	return result;
}

Example 5

Source File: KafkaRiver.java From elasticsearch-river-kafka with Apache License 2.0

6 votes

void executeBuilder(BulkRequestBuilder bulkRequestBuilder)
{
	if(bulkRequestBuilder.numberOfActions() == 0)
		return;
	
	++stats.flushes;
	BulkResponse response = bulkRequestBuilder.execute().actionGet();
	if (response.hasFailures()) {
              logger.warn("failed to execute" + response.buildFailureMessage());
          }
	
	for(BulkItemResponse resp : response){
		if(resp.isFailed()){
			stats.failed++;
		}else{
			stats.succeeded++;
		}
	}
}

Example 6

Source File: IndexingComponent.java From elasticsearch-reindex-tool with Apache License 2.0

5 votes

private Optional<BulkResult> executeBulk(int indexedCount, BulkRequestBuilder bulkRequest) {
  if (bulkRequest.numberOfActions() > 0) {
    BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
    Set<String> failedIds = Stream.of(bulkItemResponses.getItems())
        .filter(BulkItemResponse::isFailed)
        .map(BulkItemResponse::getId)
        .collect(Collectors.toSet());
    return Optional.of(new BulkResult(indexedCount, failedIds));
  }
  return Optional.empty();
}

Example 7

Source File: SearchUpdateJobImpl.java From stash-codesearch-plugin with Apache License 2.0

5 votes

@Override
public void doReindex(Client client, GitScm gitScm, GlobalSettings globalSettings) {
    if (!globalSettings.getIndexingEnabled()) {
        return;
    }
    deleteLatestIndexedNote(client);
    while (true) {
        try {
            SearchRequestBuilder esReq = client.prepareSearch(ES_UPDATEALIAS)
                .setSize(1000)
                .setFetchSource(false)
                .setRouting(getRepoDesc())
                .setQuery(filteredQuery(matchAllQuery(), andFilter(
                    sfu.projectRepositoryFilter(
                        repository.getProject().getKey(), repository.getSlug()),
                    sfu.exactRefFilter(ref))));
            BulkRequestBuilder bulkDelete = client.prepareBulk().setRefresh(true);
            for (SearchHit hit : esReq.get().getHits().getHits()) {
                bulkDelete.add(buildDeleteFromRef(client, hit.getType(), hit.getId()));
            }
            if (bulkDelete.numberOfActions() == 0) {
                break;
            }
            bulkDelete.get();
        } catch (Exception e) {
            log.error("Could not delete documents for {}, aborting", toString(), e);
            return;
        }
    }
    doUpdate(client, gitScm, globalSettings);
}

Example 8

Source File: IndexBatchBolt.java From storm-trident-elasticsearch with Apache License 2.0

5 votes

protected void bulkUpdateIndexes( ) {

        List<Tuple> inputs = new ArrayList<>(queue.size());
        queue.drainTo(inputs);
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (Tuple input : inputs) {
            Document<T> doc = mapper.map(input);
            IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource((String)doc.getSource());

            if(doc.getParentId() != null) {
                request.setParent(doc.getParentId());
            }
            bulkRequest.add(request);
        }

        try {
            if (bulkRequest.numberOfActions() > 0) {
                BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
                if (bulkItemResponses.hasFailures()) {
                    BulkItemResponse[] items = bulkItemResponses.getItems();
                    for (int i = 0; i < items.length; i++) {
                        ackOrFail(items[i], inputs.get(i));
                    }
                } else {
                    ackAll(inputs);
                }
            }
        } catch (ElasticsearchException e) {
            LOGGER.error("Unable to process bulk request, " + inputs.size() + " tuples are in failure", e);
            outputCollector.reportError(e.getRootCause());
            failAll(inputs);
        }
    }

Example 9