Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder

The following examples show how to use org.elasticsearch.action.bulk.BulkRequestBuilder. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
public void testIt() throws Exception
{
	byte[] data = "somedata".getBytes();
	List<String> l = new ArrayList<>();
	
	MessageHandler m = new RawMessageHandler();
	Message message = createMock(Message.class);
	expect(message.payload()).andReturn(ByteBuffer.wrap(data));
	
	BulkRequestBuilder bulkRequestBuilder = createMock(BulkRequestBuilder.class);
	expect(bulkRequestBuilder.add(aryEq(data), eq(0), eq(data.length), eq(false))).andReturn(null);
	replay(message, bulkRequestBuilder);
	
	m.handle(bulkRequestBuilder, message);
	verify(bulkRequestBuilder, message);
}
 
Example 2
Source Project: cassandra-river   Source File: CassandraRiver.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void run() {
	logger.info("Starting thread with {} keys", this.keys.rowColumnMap.size());
              if (closed) {
                  return;
              }
              
              BulkRequestBuilder bulk = client.prepareBulk();
              for(String key : this.keys.rowColumnMap.keySet()){
              	
  				try {
  					String id = UUID.nameUUIDFromBytes(key.getBytes()).toString();
  					bulk.add(indexRequest(this.indexName).type(this.typeName)
					    							 .id(id)
					    							 .source(this.keys.rowColumnMap.get(key)));
  				} catch (Exception e) {
  					logger.error("failed to entry to bulk indexing");
  				}
  				
              	if(bulk.numberOfActions() >= this.batchSize){
              		saveToEs(bulk);
  					bulk = client.prepareBulk();
              	}
              }
}
 
Example 3
Source Project: uavstack   Source File: InvokeChainDataCollectHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 存储到ES
 * 
 * @param appUUID
 * @param span
 * @param bulkRequest
 */
private void pushSpanToBulkRequest(String appUUID, String appGroup, Span span, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(span.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, InvokeChainIndexMgr.IVC_Table, uuid);

    Map<String, Object> m = span.toMap();

    m.put("appuuid", appUUID);
    m.put("appgroup", appGroup);

    irb.setSource(m);

    bulkRequest.add(irb);
}
 
Example 4
Source Project: canal   Source File: ES7xTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void test02() {
    Map<String, Object> esFieldData = new LinkedHashMap<>();
    esFieldData.put("userId", 2L);
    esFieldData.put("eventId", 4L);
    esFieldData.put("eventName", "网络异常");
    esFieldData.put("description", "第四个事件信息");

    Map<String, Object> relations = new LinkedHashMap<>();
    esFieldData.put("user_event", relations);
    relations.put("name", "event");
    relations.put("parent", "2");

    BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();
    bulkRequestBuilder.add(transportClient.prepareIndex("test", "osm", "2_4")
        .setRouting("2")
        .setSource(esFieldData));
    commit(bulkRequestBuilder);
}
 
Example 5
Source Project: uavstack   Source File: ThreadAnalysisCollectDataHandler.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 
 * @param appUUID
 * @param appGroup
 * @param span
 * @param bulkRequest
 */
private void pushDataToBulkRequest(JavaThreadObject jto, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(jto.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, ThreadAnalysisIndexMgr.JTA_TABLE, uuid);

    Map<String, Object> m = jto.toMap();

    irb.setSource(m);

    bulkRequest.add(irb);
}
 
Example 6
Source Project: trident-elasticsearch   Source File: IndexState.java    License: Apache License 2.0 6 votes vote down vote up
private void runBulk(BulkRequestBuilder bulkRequest) {
    if(bulkRequest.numberOfActions() > 0) {
        int tryCount = 0;
        boolean shouldTryAgain;
        do {
            shouldTryAgain = false;
            try {                
                BulkResponse bulkResponse = bulkRequest.execute().actionGet();
                if (bulkResponse.hasFailures()) {
                    shouldTryAgain = this.exceptionHandler.onBulkRequestFailure(bulkResponse, tryCount);
                    tryCount++;
                }
            } catch (ElasticsearchException e) {
                shouldTryAgain = this.exceptionHandler.onElasticSearchException(e, tryCount);
                tryCount++;
            }
        } while (shouldTryAgain);
    } else {
        LOG.debug("Empty batch being submitted");
    }
}
 
Example 7
Source Project: metacat   Source File: ElasticSearchUtilImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void updateDocs(final String type, final List<String> ids, final ObjectNode node) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            ids.forEach(id -> {
                bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                    .setRetryOnConflict(NO_OF_CONFLICT_RETRIES)
                    .setDoc(metacatJson.toJsonAsBytes(node), XContentType.JSON));
            });
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.updateDocs.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchUpdate.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.updatDocs", type, ids, e,
            Metrics.CounterElasticSearchBulkUpdate.getMetricName());
    }
}
 
Example 8
@Before
public void before() throws IOException {
  if(!client.admin().indices().prepareExists(INDEX).execute().actionGet().isExists()) {
    client.admin().indices().prepareCreate(INDEX).execute().actionGet();
  }
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  final BulkRequestBuilder bulk = client.prepareBulk();
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}
 
Example 9
Source Project: canal   Source File: ES7xTest.java    License: Apache License 2.0 6 votes vote down vote up
private void commit(BulkRequestBuilder bulkRequestBuilder) {
    if (bulkRequestBuilder.numberOfActions() > 0) {
        BulkResponse response = bulkRequestBuilder.execute().actionGet();
        if (response.hasFailures()) {
            for (BulkItemResponse itemResponse : response.getItems()) {
                if (!itemResponse.isFailed()) {
                    continue;
                }

                if (itemResponse.getFailure().getStatus() == RestStatus.NOT_FOUND) {
                    System.out.println(itemResponse.getFailureMessage());
                } else {
                    System.out.println("ES bulk commit error" + itemResponse.getFailureMessage());
                }
            }
        }
    }
}
 
Example 10
Source Project: SpringBootLearn   Source File: ElasticsearchUtil.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 批量删除
 * @Author lihaodong
 * @Description
 * @Date 20:18 2018/12/21
 * @Param [index, type, ids]
 * @return org.elasticsearch.action.bulk.BulkResponse
 **/
public static BulkResponse bulkDeleteDocument(String index, String type, List<String> ids) {
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    ids.stream().forEach(
            id -> bulkRequest.add(client.prepareDelete(index, type, id))
    );
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    return bulkResponse;
}
 
Example 11
private void sendToLocalCluster(final String scrollId, final SearchHit[] hits) {

            // prepare bulk request
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                IndexRequestBuilder builder = client.prepareIndex(toIndex,
                        toType != null ? toType : hit.getType(), hit.getId())
                        .setSource(hit.getSource());
                Map<String, SearchHitField> fields = hit.getFields();
                if (fields != null && fields.containsKey("_parent")) {
                    SearchHitField parentField = fields.get("_parent");
                    if (parentField != null) {
                        String parentId = parentField.getValue();
                        builder.setParent(parentId);
                    }
                }
                bulkRequest.add(builder);
            }

            // send bulk request, if success response got, searching the next 10 results using scroll_id
            // using this listener (inner class) to listen to results
            bulkRequest.execute(new ActionListener<BulkResponse>() {
                @Override
                public void onResponse(final BulkResponse bulkResponse) {
                    if (bulkResponse.hasFailures()) {
                        throw new ReindexingException(bulkResponse
                                .buildFailureMessage());
                    }
                    client.prepareSearchScroll(scrollId).setScroll(scroll)
                            .execute(ReindexingListener.this);
                }

                @Override
                public void onFailure(final Throwable e) {
                    ReindexingListener.this.onFailure(e);
                }
            });
        }
 
Example 12
@Test
public void testBulkIndexing() throws Exception {
    final String field = "bulk_indexing_test";
    Map<String, String> content;
    final int numDocs = 100;
    BulkRequestBuilder bulkBuilder = client().prepareBulk();
    for (int i = 0; i < numDocs; i++) {
        content = new HashMap<>();
        content.put(field, "http://domain" + i + ".com/foo" + i + "/bar.html");
        bulkBuilder.add(client().prepareIndex(INDEX, TYPE).setSource(content));
    }
    BulkResponse bulkResponse = bulkBuilder.get();
    assertThat(bulkResponse.buildFailureMessage(), bulkResponse.hasFailures(), equalTo(false));
}
 
Example 13
Source Project: summerframework   Source File: ElasticsearchTemplate.java    License: Apache License 2.0 5 votes vote down vote up
public int updateBatchData(ESBasicInfo esBasicInfo, Object object) throws IOException {
    BulkRequestBuilder bulkRequest = esClient.prepareBulk();

    for (String id : esBasicInfo.getIds()) {
        bulkRequest.add(esClient.prepareUpdate(esBasicInfo.getIndex(), esBasicInfo.getType(), id)
            .setDoc(mapper.writeValueAsString(object), XContentType.JSON));
    }

    bulkRequest.execute().actionGet();

    return bulkRequest.numberOfActions();
}
 
Example 14
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) {
    BulkResponse bulkResponse = bulkRequest.execute().actionGet();

    getLog().info("Bulk request of batch size: " + bulkRequest.numberOfActions() + " took "
            + bulkResponse.getTookInMillis() + " ms in index builder: " + getName());

    for (BulkItemResponse response : bulkResponse.getItems()) {
        if (response.getResponse() instanceof DeleteResponse) {
            DeleteResponse deleteResponse = response.getResponse();

            if (response.isFailed()) {
                getLog().error("Problem deleting doc: " + response.getId() + " in index builder: " + getName()
                        + " error: " + response.getFailureMessage());
            } else if (!deleteResponse.isFound()) {
                getLog().debug("ES could not find a doc with id: " + deleteResponse.getId()
                        + " to delete in index builder: " + getName());
            } else {
                getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder: "
                        + getName());
            }
        } else if (response.getResponse() instanceof IndexResponse) {
            IndexResponse indexResponse = response.getResponse();

            if (response.isFailed()) {
                getLog().error("Problem updating content for doc: " + response.getId() + " in index builder: "
                        + getName() + " error: " + response.getFailureMessage());
            } else {
                getLog().debug("ES indexed content for doc with id: " + indexResponse.getId()
                        + " in index builder: " + getName());
            }
        }
    }
}
 
Example 15
Source Project: hawkular-apm   Source File: TraceServiceElasticsearch.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void storeFragments(String tenantId, List<Trace> traces)
        throws StoreException {
    client.initTenant(tenantId);

    BulkRequestBuilder bulkRequestBuilder = client.getClient().prepareBulk();

    try {
        for (int i = 0; i < traces.size(); i++) {
            Trace trace = traces.get(i);
            String json = mapper.writeValueAsString(trace);

            if (msgLog.isTraceEnabled()) {
                msgLog.tracef("Storing trace: %s", json);
            }

            bulkRequestBuilder.add(client.getClient().prepareIndex(client.getIndex(tenantId),
                    TRACE_TYPE, trace.getFragmentId()).setSource(json));
        }
    } catch (JsonProcessingException e) {
        throw new StoreException(e);
    }

    BulkResponse bulkItemResponses = bulkRequestBuilder.execute().actionGet();

    if (bulkItemResponses.hasFailures()) {

        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Failed to store traces to elasticsearch: "
                    + bulkItemResponses.buildFailureMessage());
        }

        throw new StoreException(bulkItemResponses.buildFailureMessage());

    } else {
        if (msgLog.isTraceEnabled()) {
            msgLog.trace("Success storing traces to elasticsearch");
        }
    }
}
 
Example 16
Source Project: baleen   Source File: Elasticsearch.java    License: Apache License 2.0 5 votes vote down vote up
private <T> void save(
    final String index,
    final String type,
    final Collection<T> collection,
    final Function<T, String> idGenerator,
    final Function<T, String> parentIdGenerator) {

  if (collection.isEmpty()) {
    return;
  }

  try {
    final BulkRequestBuilder bulkBuilder = esResource.getClient().prepareBulk();

    for (final T t : collection) {
      final String source = MAPPER.writeValueAsString(t);
      bulkBuilder.add(
          esResource
              .getClient()
              .prepareIndex(index, type, idGenerator.apply(t))
              .setSource(source, XContentType.JSON)
              .setParent(parentIdGenerator.apply(t)));
    }

    final RestStatus status = bulkBuilder.execute().actionGet().status();
    if (!status.equals(RestStatus.OK)) {
      throw new BaleenException("Rest call failed with code " + status.getStatus());
    }
  } catch (final Exception ee) {
    getMonitor().error("Couldn't persist to Elasticsearch", ee);
  }
}
 
Example 17
protected void bulkUpdateIndexes( ) {

        List<Tuple> inputs = new ArrayList<>(queue.size());
        queue.drainTo(inputs);
        BulkRequestBuilder bulkRequest = client.prepareBulk();
        for (Tuple input : inputs) {
            Document<T> doc = mapper.map(input);
            IndexRequestBuilder request = client.prepareIndex(doc.getName(), doc.getType(), doc.getId()).setSource((String)doc.getSource());

            if(doc.getParentId() != null) {
                request.setParent(doc.getParentId());
            }
            bulkRequest.add(request);
        }

        try {
            if (bulkRequest.numberOfActions() > 0) {
                BulkResponse bulkItemResponses = bulkRequest.execute().actionGet();
                if (bulkItemResponses.hasFailures()) {
                    BulkItemResponse[] items = bulkItemResponses.getItems();
                    for (int i = 0; i < items.length; i++) {
                        ackOrFail(items[i], inputs.get(i));
                    }
                } else {
                    ackAll(inputs);
                }
            }
        } catch (ElasticsearchException e) {
            LOGGER.error("Unable to process bulk request, " + inputs.size() + " tuples are in failure", e);
            outputCollector.reportError(e.getRootCause());
            failAll(inputs);
        }
    }
 
Example 18
Source Project: vertexium   Source File: BulkUpdateItem.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
@Override
public void addToBulkRequest(Client client, BulkRequestBuilder bulkRequestBuilder) {
    UpdateRequestBuilder updateRequestBuilder = client
        .prepareUpdate(getIndexName(), getType(), getDocumentId());
    if (!updateOnly) {
        updateRequestBuilder = updateRequestBuilder
            .setScriptedUpsert(true)
            .setUpsert((Map<String, Object>) (Map) source);
    }
    UpdateRequest updateRequest = updateRequestBuilder
        .setScript(new Script(
            ScriptType.STORED,
            null,
            "updateFieldsOnDocumentScript",
            ImmutableMap.of(
                "fieldsToSet", fieldsToSet.entrySet().stream()
                    .collect(Collectors.toMap(
                        Map.Entry::getKey,
                        entry -> new ArrayList<>(entry.getValue())
                    )),
                "fieldsToRemove", new ArrayList<>(fieldsToRemove),
                "fieldsToRename", fieldsToRename,
                "additionalVisibilities", new ArrayList<>(additionalVisibilities),
                "additionalVisibilitiesToDelete", new ArrayList<>(additionalVisibilitiesToDelete)
            )
        ))
        .setRetryOnConflict(Elasticsearch7SearchIndex.MAX_RETRIES)
        .request();
    bulkRequestBuilder.add(updateRequest);
}
 
Example 19
Source Project: elasticsearch-jest-example   Source File: TransportClient.java    License: MIT License 5 votes vote down vote up
/**
 * 删除查询到的文档
 * @param index
 * @param name
 * @param value
 */
private static void scrollSearchDelete(String index,String name,String value){
	Client client = createTransportClient();
	QueryBuilder qb = termQuery(name, value);
	SearchResponse scrollResp = client.prepareSearch(index)
			.setSearchType(SearchType.SCAN)
			.setScroll(new TimeValue(60000))
			.setQuery(qb)
			.setSize(100).execute().actionGet(); //100 hits per shard will be returned for each scroll

	BulkRequestBuilder bulkRequest = client.prepareBulk();

	while (true) {
		for (SearchHit hit : scrollResp.getHits().getHits()) {
			bulkRequest.add(client.prepareDelete(hit.getIndex(),hit.getType(),hit.getId()));
		}
		scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(600000)).execute().actionGet();
		if (scrollResp.getHits().getHits().length == 0) {
			break;
		}
	}
	BulkResponse bulkResponse = bulkRequest.execute().actionGet();
	if (bulkResponse.hasFailures()) {
		BulkItemResponse[] bulkItemResponse = bulkResponse.getItems();
		for (int i = 0; i <bulkItemResponse.length ; i++) {
			System.out.println(bulkItemResponse[i].getItemId()+":"+bulkItemResponse[i].getIndex()+":"+bulkItemResponse[i].getFailureMessage());
		}
	}
}
 
Example 20
Source Project: AsuraFramework   Source File: EsDoc.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 批量save
 * @param indexName
 * @param indexType
 * @param list
 * @return
 */
private  BulkResponse bulkSaveOrUpdate(String indexName, String indexType, List<DocPo> list){
    TransportClient client = esClientFactory.getClient();
    BulkResponse bulkResponse=null;
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for(int i=0;i<list.size();i++){
        bulkRequest.add(new IndexRequest(esClientFactory.getIndexs(indexName), indexType,list.get(i).getId()).source(JSON.toJSONString(list.get(i).getObj())));
    }
    bulkResponse = bulkRequest.get();
    // 处理错误信息
    handBulkResponseException(bulkResponse);
    return bulkResponse;
}
 
Example 21
Source Project: AsuraFramework   Source File: EsDoc.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * 批量del
 * @param indexName
 * @param indexType
 * @param
 * @return
 */
private  BulkResponse bulkDelete(String indexName, String indexType, List<String> ids){
    TransportClient client = esClientFactory.getClient();
    BulkResponse bulkResponse=null;
    BulkRequestBuilder bulkRequest = client.prepareBulk();
    for(int i=0;i<ids.size();i++){
        bulkRequest.add(new DeleteRequest(esClientFactory.getIndexs(indexName), indexType,ids.get(i) ));
    }
    bulkResponse = bulkRequest.get();
    // 处理错误信息
    handBulkResponseException(bulkResponse);
    return bulkResponse;
}
 
Example 22
Source Project: fess   Source File: EsAbstractBehavior.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected int delegateQueryDelete(final ConditionBean cb, final DeleteOption<? extends ConditionBean> option) {
    final SearchRequestBuilder builder = client.prepareSearch(asEsIndex()).setScroll(scrollForDelete).setSize(sizeForDelete);
    final EsAbstractConditionBean esCb = (EsAbstractConditionBean) cb;
    if (esCb.getPreference() != null) {
        esCb.setPreference(esCb.getPreference());
    }
    esCb.request().build(builder);
    SearchResponse response = esCb.build(builder).execute().actionGet(scrollSearchTimeout);
    String scrollId = response.getScrollId();
    int count = 0;
    try {
        while (scrollId != null) {
            final SearchHits searchHits = getSearchHits(response);
            final SearchHit[] hits = searchHits.getHits();
            if (hits.length == 0) {
                break;
            }

            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            for (final SearchHit hit : hits) {
                bulkRequest.add(client.prepareDelete().setIndex(asEsIndex()).setId(hit.getId()));
            }
            count += hits.length;
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(bulkTimeout);
            if (bulkResponse.hasFailures()) {
                throw new IllegalBehaviorStateException(bulkResponse.buildFailureMessage());
            }

            response = client.prepareSearchScroll(scrollId).setScroll(scrollForDelete).execute().actionGet(scrollSearchTimeout);
            if (!scrollId.equals(response.getScrollId())) {
                deleteScrollContext(scrollId);
            }
        }
    } finally {
        deleteScrollContext(scrollId);
    }
    return count;
}
 
Example 23
@Override
public final void write(List<? extends Document> documents) throws Exception {
    BulkRequestBuilder bulkRequest = client.prepareBulk();

    for (Document doc : documents) {
        XContentParser parser = null;
        parser = XContentFactory.xContent(XContentType.JSON)
                .createParser(NamedXContentRegistry.EMPTY,
                        DeprecationHandler.THROW_UNSUPPORTED_OPERATION,
                        doc.getOutputData().getBytes());
        parser.close();
        XContentBuilder builder = jsonBuilder().copyCurrentStructure(parser);


        IndexRequestBuilder request = client.prepareIndex(
                indexName,
                typeName).setSource(
                builder);
        request.setId(doc.getPrimaryKeyFieldValue());
        bulkRequest.add(request);
    }
    //check that no nonessential processes failed
    if (documents.size() != 0) {
        BulkResponse response;
        response = bulkRequest.execute().actionGet(timeout);
        getResponses(response);
    }
}
 
Example 24
Source Project: foxtrot   Source File: TableMapStore.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void storeAll(Map<String, Table> map) {
    if(map == null) {
        throw new TableMapStoreException("Illegal Store Request - Null Map");
    }
    if(map.containsKey(null)) {
        throw new TableMapStoreException("Illegal Store Request - Null Key is Present");
    }

    logger.info("Store all called for multiple values");
    BulkRequestBuilder bulkRequestBuilder = elasticsearchConnection.getClient()
            .prepareBulk()
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
    for(Map.Entry<String, Table> mapEntry : map.entrySet()) {
        try {
            if(mapEntry.getValue() == null) {
                throw new TableMapStoreException(
                        String.format("Illegal Store Request - Object is Null for Table - %s", mapEntry.getKey()));
            }
            Map<String, Object> sourceMap = ElasticsearchQueryUtils.toMap(objectMapper, mapEntry.getValue());
            bulkRequestBuilder.add(elasticsearchConnection.getClient()
                                           .prepareIndex(TABLE_META_INDEX, TABLE_META_TYPE, mapEntry.getKey())
                                           .setSource(sourceMap));
        } catch (Exception e) {
            throw new TableMapStoreException("Error bulk saving meta: ", e);
        }
    }
    bulkRequestBuilder.execute()
            .actionGet();
}
 
Example 25
@SuppressWarnings("rawtypes")
private BulkResponse writeAcl(ACLDocumentOperation operation, Collection<SearchGuardACLDocument> docs) throws Exception {
    BulkRequestBuilder builder = client.getClient().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    BulkRequest request = operation.buildRequest(this.client.getClient(), builder, docs);
    client.addCommonHeaders();
    return this.client.getClient().bulk(request).actionGet();
}
 
Example 26
Source Project: vertexium   Source File: BulkUpdateItem.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void addToBulkRequest(Client client, BulkRequestBuilder bulkRequestBuilder) {
    UpdateRequestBuilder updateRequestBuilder = client
        .prepareUpdate(getIndexName(), getType(), getDocumentId());
    if (!updateOnly) {
        updateRequestBuilder = updateRequestBuilder
            .setScriptedUpsert(true)
            .setUpsert(source);
    }
    UpdateRequest updateRequest = updateRequestBuilder
        .setScript(new Script(
            ScriptType.STORED,
            "painless",
            "updateFieldsOnDocumentScript",
            ImmutableMap.of(
                "fieldsToSet", fieldsToSet.entrySet().stream()
                    .collect(Collectors.toMap(
                        Map.Entry::getKey,
                        entry -> new ArrayList<>(entry.getValue())
                    )),
                "fieldsToRemove", new ArrayList<>(fieldsToRemove),
                "fieldsToRename", fieldsToRename,
                "additionalVisibilities", new ArrayList<>(additionalVisibilities),
                "additionalVisibilitiesToDelete", new ArrayList<>(additionalVisibilitiesToDelete)
            )
        ))
        .setRetryOnConflict(Elasticsearch5SearchIndex.MAX_RETRIES)
        .request();
    bulkRequestBuilder.add(updateRequest);
}
 
Example 27
Source Project: metacat   Source File: ElasticSearchUtilImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void softDeleteDoc(
    final String type,
    final List<String> ids,
    final MetacatRequestContext metacatRequestContext) {
    try {
        RETRY_ES_PUBLISH.call(() -> {
            final BulkRequestBuilder bulkRequest = client.prepareBulk();
            final XContentBuilder builder = XContentFactory.contentBuilder(contentType);
            builder.startObject().field(ElasticSearchDoc.Field.DELETED, true)
                .field(ElasticSearchDoc.Field.TIMESTAMP, java.time.Instant.now().toEpochMilli())
                .field(ElasticSearchDoc.Field.USER, metacatRequestContext.getUserName()).endObject();
            ids.forEach(id -> bulkRequest.add(client.prepareUpdate(esIndex, type, id)
                .setRetryOnConflict(NO_OF_CONFLICT_RETRIES).setDoc(builder)));
            final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
            if (bulkResponse.hasFailures()) {
                for (BulkItemResponse item : bulkResponse.getItems()) {
                    if (item.isFailed()) {
                        handleException("ElasticSearchUtil.bulkSoftDelete.item", type, item.getId(),
                            item.getFailure().getCause(), Metrics.CounterElasticSearchDelete.getMetricName());
                    }
                }
            }
            return null;
        });
    } catch (Exception e) {
        handleException("ElasticSearchUtil.bulkSoftDelete", type, ids, e,
            Metrics.CounterElasticSearchBulkDelete.getMetricName());
    }
}
 
Example 28
Source Project: metacat   Source File: ElasticSearchUtilImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Bulk save of the entities.
 *
 * @param type index type
 * @param docs metacat documents
 */
private void bulkSaveToIndex(final String type, final List<ElasticSearchDoc> docs, final String index) {
    if (docs != null && !docs.isEmpty()) {
        try {

            RETRY_ES_PUBLISH.call(() -> {
                final BulkRequestBuilder bulkRequest = client.prepareBulk();
                for (ElasticSearchDoc doc : docs) {
                    final IndexRequestBuilder indexRequestBuilder = prepareIndexRequest(index, type, doc);
                    if (indexRequestBuilder != null) {
                        bulkRequest.add(indexRequestBuilder);
                    }
                }

                if (bulkRequest.numberOfActions() > 0) {
                    final BulkResponse bulkResponse = bulkRequest.execute().actionGet(esBulkCallTimeout);
                    log.info("Bulk saving metadata of index {} type {} with size {}.",
                        index, type, docs.size());
                    if (bulkResponse.hasFailures()) {
                        for (BulkItemResponse item : bulkResponse.getItems()) {
                            if (item.isFailed()) {
                                handleException("ElasticSearchUtil.bulkSaveToIndex.index", type, item.getId(),
                                    item.getFailure().getCause(), Metrics.CounterElasticSearchSave.getMetricName());
                            }
                        }
                    }
                }
                return null;
            });
        } catch (Exception e) {
            final List<String> docIds = docs.stream().map(ElasticSearchDoc::getId).collect(Collectors.toList());
            handleException("ElasticSearchUtil.bulkSaveToIndex", type, docIds, e,
                Metrics.CounterElasticSearchBulkSave.getMetricName());
        }
    }
}
 
Example 29
Source Project: javabase   Source File: UpdateMappingFieldDemo.java    License: Apache License 2.0 5 votes vote down vote up
private static void reindexData(IndicesAdminClient indicesAdminClient) {
    //查询原来的所有数据,TimeValue是需要保存的时长
    SearchResponse searchResponse = client.prepareSearch(ALIX_NAME).setTypes(INDEX_TYPE).setQuery(QueryBuilders.matchAllQuery()).
            setSearchType(SearchType.SCAN).setScroll(new TimeValue(20000))
            .setSize(100).execute().actionGet();
    //当前id
         String scrollId = searchResponse.getScrollId();
        while(StringUtils.isNotEmpty(scrollId)) {
            BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
            SearchResponse scrollSearchResponse = client.prepareSearchScroll(scrollId).setScroll(new TimeValue(20000)).execute().actionGet();
            SearchHits hits = scrollSearchResponse.getHits();
            if(hits.getHits().length>0) {
                for (SearchHit searchHitFields : hits.getHits()) {
                    IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v2, INDEX_TYPE).setId(searchHitFields.getId()).setSource(searchHitFields.getSource()).setOpType(IndexRequest.OpType.INDEX);
                    bulkRequestBuilder.add(indexRequestBuilder);
                }
                BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
                if (bulkResponse.hasFailures()) {
                    log.error("reindex失败 : {}", bulkResponse.buildFailureMessage());
                } else {
                    log.info("reindex {}条成功:", hits.getHits().length);
                }
            }else{
                break;
            }
        }
}
 
Example 30
Source Project: fess   Source File: BsDuplicateHostBhv.java    License: Apache License 2.0 4 votes vote down vote up
public int[] batchDelete(List<DuplicateHost> list, RequestOptionCall<BulkRequestBuilder> call,
        RequestOptionCall<IndexRequestBuilder> entityCall) {
    return doBatchDelete(new BulkList<>(list, call, entityCall), null);
}