Java Code Examples for org.elasticsearch.action.bulk.BulkRequestBuilder

The following are top voted examples for showing how to use org.elasticsearch.action.bulk.BulkRequestBuilder. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: Elasticsearch-Tutorial-zh-CN   File: BaseDemo.java   View source code 9 votes vote down vote up
/**
 * 批量删除
 *
 * @param transportClient
 */
private static void batchDelete(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	DeleteRequestBuilder deleteRequestBuilder1 = transportClient.prepareDelete("product_index", "product", "1");
	DeleteRequestBuilder deleteRequestBuilder2 = transportClient.prepareDelete("product_index", "product", "2");
	DeleteRequestBuilder deleteRequestBuilder3 = transportClient.prepareDelete("product_index", "product", "3");

	bulkRequestBuilder.add(deleteRequestBuilder1);
	bulkRequestBuilder.add(deleteRequestBuilder2);
	bulkRequestBuilder.add(deleteRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}

}
 
Example 2
Project: para-search-elasticsearch   File: ElasticSearch.java   View source code 7 votes vote down vote up
@Override
public <P extends ParaObject> void indexAll(String appid, List<P> objects) {
	if (StringUtils.isBlank(appid) || objects == null || objects.isEmpty()) {
		return;
	}
	BulkRequestBuilder brb = client().prepareBulk();
	for (ParaObject po : objects) {
		brb.add(client().prepareIndex(getIndexName(appid), getType(), po.getId()).
				setSource(ElasticSearchUtils.getSourceFromParaObject(po)));
	}
	if (brb.numberOfActions() > 0) {
		ActionListener<BulkResponse> responseHandler = ElasticSearchUtils.getBulkIndexResponseHandler();
		if (isAsyncEnabled()) {
			brb.execute(responseHandler);
		} else {
			responseHandler.onResponse(brb.execute().actionGet());
		}
	}
	logger.debug("Search.indexAll() {}", objects.size());
}
 
Example 3
Project: para-search-elasticsearch   File: ElasticSearch.java   View source code 7 votes vote down vote up
@Override
public <P extends ParaObject> void unindexAll(String appid, List<P> objects) {
	if (StringUtils.isBlank(appid) || objects == null || objects.isEmpty()) {
		return;
	}
	BulkRequestBuilder brb = client().prepareBulk();
	for (ParaObject po : objects) {
		brb.add(client().prepareDelete(getIndexName(appid), getType(), po.getId()));
	}
	if (brb.numberOfActions() > 0) {
		ActionListener<BulkResponse> responseHandler = ElasticSearchUtils.getBulkIndexResponseHandler();
		if (isAsyncEnabled()) {
			brb.execute(responseHandler);
		} else {
			responseHandler.onResponse(brb.execute().actionGet());
		}
	}
	logger.debug("Search.unindexAll() {}", objects.size());
}
 
Example 4
Project: database-transform-tool   File: ElasticsearchTransportFactory.java   View source code 7 votes vote down vote up
@Override
public String bulkDelete(String index, String type, String... ids) {
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (String id : ids) {
			bulkRequest.add(client.prepareDelete(index, type, id));
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 
Example 5
Project: uavstack   File: InvokeChainDataCollectHandler.java   View source code 7 votes vote down vote up
/**
 * 存储到ES
 * 
 * @param appUUID
 * @param span
 * @param bulkRequest
 */
private void pushSpanToBulkRequest(String appUUID, String appGroup, Span span, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(span.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, InvokeChainIndexMgr.IVC_Table, uuid);

    Map<String, Object> m = span.toMap();

    m.put("appuuid", appUUID);
    m.put("appgroup", appGroup);

    irb.setSource(m);

    bulkRequest.add(irb);
}
 
Example 6
Project: incubator-skywalking   File: BatchEsDAO.java   View source code 7 votes vote down vote up
@Override public void batchPersistence(List<?> batchCollection) {
    BulkRequestBuilder bulkRequest = getClient().prepareBulk();

    logger.debug("bulk data size: {}", batchCollection.size());
    if (CollectionUtils.isNotEmpty(batchCollection)) {
        batchCollection.forEach(builder -> {
            if (builder instanceof IndexRequestBuilder) {
                bulkRequest.add((IndexRequestBuilder)builder);
            }
            if (builder instanceof UpdateRequestBuilder) {
                bulkRequest.add((UpdateRequestBuilder)builder);
            }
        });

        BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        if (bulkResponse.hasFailures()) {
            logger.error(bulkResponse.buildFailureMessage());
        }
    }
}
 
Example 7
Project: elasticsearch_my   File: WaitUntilRefreshIT.java   View source code 6 votes vote down vote up
public void testBulk() {
    // Index by bulk with RefreshPolicy.WAIT_UNTIL
    BulkRequestBuilder bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareIndex("test", "test", "1").setSource("foo", "bar"));
    assertBulkSuccess(bulk.get());
    assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get(), "1");

    // Update by bulk with RefreshPolicy.WAIT_UNTIL
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareUpdate("test", "test", "1").setDoc(Requests.INDEX_CONTENT_TYPE, "foo", "baz"));
    assertBulkSuccess(bulk.get());
    assertSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "baz")).get(), "1");

    // Delete by bulk with RefreshPolicy.WAIT_UNTIL
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareDelete("test", "test", "1"));
    assertBulkSuccess(bulk.get());
    assertNoSearchHits(client().prepareSearch("test").setQuery(matchQuery("foo", "bar")).get());

    // Update makes a noop
    bulk = client().prepareBulk().setRefreshPolicy(RefreshPolicy.WAIT_UNTIL);
    bulk.add(client().prepareDelete("test", "test", "1"));
    assertBulkSuccess(bulk.get());
}
 
Example 8
Project: css-elasticsearch   File: ElasticsearchTransportFactory.java   View source code 6 votes vote down vote up
public String bulkUpsert(String index,String type,List<Object> jsons){
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (Object json : jsons) {
			JSONObject obj = JSON.parseObject(JSON.toJSONString(json));
			String id = UUIDs.base64UUID();
			if(obj.containsKey("id")){
				id = obj.getString("id");
				obj.remove("id");
				bulkRequest.add(client.prepareUpdate(index, type, id).setDoc(obj.toJSONString(),XContentType.JSON));
			}else{
				bulkRequest.add(client.prepareIndex(index, type, id).setSource(obj.toJSONString(),XContentType.JSON));
			}
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 
Example 9
Project: css-elasticsearch   File: ElasticsearchTransportFactory.java   View source code 6 votes vote down vote up
@Override
public String bulkDelete(String index, String type, String... ids) {
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (String id : ids) {
			bulkRequest.add(client.prepareDelete(index, type, id));
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 
Example 10
Project: escommons   File: AbstractEsRepository.java   View source code 6 votes vote down vote up
@Nonnull
@Override
public List<T> save(@Nonnull final List<T> documents, @Nonnull final String indexName) {
    Assert.notNull(documents, "The list of documents should not be null");
    assertIndexNameNotNull(indexName);
    final BulkRequestBuilder bulkBuilder = esClient.prepareBulk();
    documents.forEach(document -> {
        final IndexRequestBuilder indexRequestBuilder = esClient
                .prepareIndex(indexName, getDocumentType())
                .setId(document.getUuid())
                .setSource(jsonComponent.serialize(document, clazz));
        bulkBuilder.add(indexRequestBuilder);
    });
    bulkBuilder.get();
    return documents;
}
 
Example 11
Project: hsweb-learning   File: ElasticSearch.java   View source code 6 votes vote down vote up
private static void BulkIndex(Client client) throws IOException {

        BulkRequestBuilder requestBuilder = client.prepareBulk();

        requestBuilder.add(client.prepareIndex("twitter","tweet","4")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","niekaijie")
                        .field("school","beiyou")
                        .field("address","haidianqu")
                        .endObject())
        );
        requestBuilder.add(client.prepareIndex("twitter","tweet","3")
                .setSource(jsonBuilder()
                        .startObject()
                        .field("user","林志颖aa")
                        .field("school","台湾大学")
                        .field("address","台湾")
                        .endObject())
        );
        BulkResponse response = requestBuilder.get();


    }
 
Example 12
Project: elasticsearch-indexing-proxy   File: RequestUtils.java   View source code 6 votes vote down vote up
public static BulkRequestBuilder createBulkRequest(final Client client, final StreamInput streamInput, final String index)
        throws IOException {
    final BulkRequestBuilder builder = client.prepareBulk();
    final BulkRequest request = builder.request();
    request.readFrom(streamInput);
    if (index != null) {
        request.requests().stream().forEach(req -> {
            if (req instanceof DeleteRequest) {
                ((DeleteRequest) req).index(index);
            } else if (req instanceof DeleteByQueryRequest) {
                ((DeleteByQueryRequest) req).indices(index);
            } else if (req instanceof IndexRequest) {
                ((IndexRequest) req).index(index);
            } else if (req instanceof UpdateRequest) {
                ((UpdateRequest) req).index(index);
            } else if (req instanceof UpdateByQueryRequest) {
                ((UpdateByQueryRequest) req).indices(index);
            } else {
                throw new ElasticsearchException("Unsupported request in bulk: " + req);
            }
        });
    }
    return builder;
}
 
Example 13
Project: Elasticsearch-Tutorial-zh-CN   File: BaseDemo.java   View source code 6 votes vote down vote up
/**
 * 批量更新
 *
 * @param transportClient
 */
private static void batchUpdate(TransportClient transportClient) throws IOException {
	BulkRequestBuilder bulkRequestBuilder = transportClient.prepareBulk();

	UpdateRequestBuilder updateRequestBuilder1 = transportClient.prepareUpdate("product_index", "product", "1")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称1")
					.endObject());

	UpdateRequestBuilder updateRequestBuilder2 = transportClient.prepareUpdate("product_index", "product", "2")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称2")
					.endObject());

	UpdateRequestBuilder updateRequestBuilder3 = transportClient.prepareUpdate("product_index", "product", "3")
			.setDoc(XContentFactory.jsonBuilder()
					.startObject()
					.field("product_name", "更新后的商品名称3")
					.endObject());


	bulkRequestBuilder.add(updateRequestBuilder1);
	bulkRequestBuilder.add(updateRequestBuilder2);
	bulkRequestBuilder.add(updateRequestBuilder3);

	BulkResponse bulkResponse = bulkRequestBuilder.get();
	for (BulkItemResponse bulkItemResponse : bulkResponse.getItems()) {
		logger.info("--------------------------------version= " + bulkItemResponse.getVersion());
	}
}
 
Example 14
Project: database-transform-tool   File: ElasticsearchTransportFactory.java   View source code 6 votes vote down vote up
public String bulkUpsert(String index,String type,List<Object> jsons){
	try {
		if(client==null){
			init();
		}
		BulkRequestBuilder bulkRequest = client.prepareBulk();
		for (Object json : jsons) {
			JSONObject obj = JSON.parseObject(JSON.toJSONString(json));
			String id = UUIDs.base64UUID();
			if(obj.containsKey("id")){
				id = obj.getString("id");
				obj.remove("id");
				bulkRequest.add(client.prepareUpdate(index, type, id).setDoc(obj.toJSONString(),XContentType.JSON));
			}else{
				bulkRequest.add(client.prepareIndex(index, type, id).setSource(obj.toJSONString(),XContentType.JSON));
			}
		}
		BulkResponse result = bulkRequest.execute().get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 
Example 15
Project: elasticsearch   File: App.java   View source code 6 votes vote down vote up
private static void deleteES(Client client) {
  BulkRequestBuilder bulkRequest = client.prepareBulk();
  SearchResponse response = client.prepareSearch(index).setTypes(type)
      .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
      .setQuery(QueryBuilders.matchAllQuery())
      .setFrom(0).setSize(20).setExplain(true).execute().actionGet();
  System.out.println("length: " + response.getHits().getHits().length);
  if (response.getHits().getHits().length != 0) {
    for (SearchHit hit : response.getHits()) {
      String id = hit.getId();
      System.out.println("id: " + id);
      bulkRequest.add(client.prepareDelete(index, type, id).request());
    }
    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
      for (BulkItemResponse item : bulkResponse.getItems()) {
        System.out.println(item.getFailureMessage());
      }
    } else {
      System.out.println("delete ok");
    }
  } else {
    System.out.println("delete ok");
  }
}
 
Example 16
Project: uavstack   File: ThreadAnalysisCollectDataHandler.java   View source code 6 votes vote down vote up
/**
 * TODO:线程分析数据的存储
 * 
 * @param result
 */
private void insertThreadObjectToES(List<JavaThreadObject> result) {

    if (this.log.isDebugEnable()) {
        this.log.debug(this, "ThreadObjectToES is :" + JSONHelper.toString(result));
    }

    BulkRequestBuilder bulkRequest = client.getClient().prepareBulk();

    for (JavaThreadObject item : result) {
        pushDataToBulkRequest(item, bulkRequest);
    }

    BulkResponse bulkResponse = bulkRequest.get();
    if (bulkResponse.hasFailures()) {
        log.err(this, "INSERT ThreadAnalysis Data to ES FAIL: " + bulkResponse.buildFailureMessage());
    }
}
 
Example 17
Project: uavstack   File: ThreadAnalysisCollectDataHandler.java   View source code 6 votes vote down vote up
/**
 * 
 * @param appUUID
 * @param appGroup
 * @param span
 * @param bulkRequest
 */
private void pushDataToBulkRequest(JavaThreadObject jto, BulkRequestBuilder bulkRequest) {

    /**
     * 保证不会重复
     */
    String uuid = EncodeHelper.encodeMD5(jto.toString());

    /**
     * 获取当前正在使用的index名称
     */
    String currentIndex = indexMgr.prepareIndex();

    IndexRequestBuilder irb = client.getClient().prepareIndex(currentIndex, ThreadAnalysisIndexMgr.JTA_TABLE, uuid);

    Map<String, Object> m = jto.toMap();

    irb.setSource(m);

    bulkRequest.add(irb);
}
 
Example 18
Project: dooo   File: IndexManager.java   View source code 6 votes vote down vote up
/**
 * 创建索引
 *
 * @param datas
 * @param indexType
 */
public static <T extends IdAble> void createIndex(List<T> datas, IndexType indexType) {
    // 批量处理request
    BulkRequestBuilder bulkRequest = EsClientManager.getClient().prepareBulk();
    byte[] json;
    for (T t : datas) {
        json = EsDataUtil.toBytes(t);
        bulkRequest.add(new IndexRequest(esprop.getIndice(), indexType.getDataName(), t.getId() + "").source(json));
    }
    // 执行批量处理request
    BulkResponse bulkResponse = bulkRequest.get();
    // 处理错误信息
    if (bulkResponse.hasFailures()) {
        LOGGER.warn("====================批量创建索引过程中出现错误 下面是错误信息==========================");
        long count = 0L;
        for (BulkItemResponse bulkItemResponse : bulkResponse) {
            LOGGER.warn("类型 " + indexType.getDataName() + " 发生错误的 索引id为 : " + bulkItemResponse.getId() + " ,错误信息为:" + bulkItemResponse.getFailureMessage());
            count++;
        }
        LOGGER.warn("====================批量创建索引过程中出现错误 上面是错误信息 共有: " + count + " 条记录==========================");
    }
}
 
Example 19
Project: SkyEye   File: Indexer.java   View source code 6 votes vote down vote up
@Override
public void doJob(EventLog log, LogDto logDto, BulkRequestBuilder bulkRequest) {
    // 进行索引(for kibana),包含api调用、第三方调用、中间件调用
    if (this.getTypes().indexOf(log.getEventType()) != -1 && (log.getStatus().equals(EventLog.MONITOR_STATUS_FAILED) || log.getStatus().equals(EventLog.MONITOR_STATUS_SUCCESS))) {
        try {
            bulkRequest.add(transportClient.prepareIndex(this.esProperties.getIndex(), this.esProperties.getDoc())
                    .setSource(this.buildXContentBuilder(log, logDto)));
        } catch (IOException e) {
            LOGGER.error("构造一条es入库数据失败, {]", logDto);
        }
    }

    // 进行后续的处理
    if (null != this.getNextJob()) {
        this.getNextJob().doJob(log, logDto, bulkRequest);
    }
}
 
Example 20
Project: jaf-examples   File: IndexUpdate.java   View source code 6 votes vote down vote up
public static void bulkUpdate(Client client ) throws IOException, InterruptedException, ExecutionException {
	
	BulkRequestBuilder req = client.prepareBulk();
	req.add(client.prepareIndex("search_test","article","2")
				.setSource(jsonBuilder()
						.startObject()
						.field("title","bulk title 02 04-05")));
	req.add(client.prepareIndex("search_test","article","3")
			.setSource(jsonBuilder()
					.startObject()
					.field("title","bulk title 03 04-05")));
	req.add(client.prepareIndex("search_test","article","4")
			.setSource(jsonBuilder()
					.startObject()
					.field("title","bulk title 04 04-05")));
       BulkResponse res = req.execute().actionGet();		
       
       if (res.hasFailures()){
       	System.out.println("Error");
       } else {
       	System.out.println("Done");
       }
       
	
	client.close();
}
 
Example 21
Project: jaf-examples   File: DocumentTests.java   View source code 6 votes vote down vote up
@Test
public void updateWithBulkTest() throws IOException {
	BulkRequestBuilder request = client.prepareBulk();
	
	request.add(client.prepareUpdate(INDEX_NAME, TYPE_NAME, "AVQcXWKy50MnK8WFflRS")
						.setDoc(jsonBuilder()
								.startObject()
								.field("name", "testUpdateBulk")
								.endObject()
						)
				);
	request.add(client.prepareUpdate(INDEX_NAME, TYPE_NAME, "AVQcU5cD50MnK8WFflRQ")
						.setDoc(jsonBuilder()
								.startObject()
								.field("name", "testUpdateBulk2")
								.endObject()
						)
				);
	BulkResponse response = request.execute().actionGet();
	Assert.assertFalse(response.hasFailures());
}
 
Example 22
Project: orientdb-elasticsearch   File: OElasticSearchDatabaseSync.java   View source code 6 votes vote down vote up
public void dropClass(final String className) {
  SearchResponse scrollResponse = esClient.getClient().prepareSearch(getIndexName()).setTypes(className)
      .setSearchType(SearchType.SCAN).setScroll(new TimeValue(60000)).setQuery(QueryBuilders.matchAllQuery()).setSize(100).get();
  final BulkRequestBuilder bulkRequestBuilder = esClient.getClient().prepareBulk().setRefresh(true);
  while (true) {
    if (scrollResponse.getHits().getHits().length == 0) {
      break;
    }

    for (SearchHit hit : scrollResponse.getHits().getHits()) {
      bulkRequestBuilder.add(esClient.getClient().prepareDelete(getIndexName(), className, hit.getId()));
    }

    scrollResponse = esClient.getClient().prepareSearchScroll(scrollResponse.getScrollId()).setScroll(new TimeValue(60000)).get();
  }
  if (bulkRequestBuilder.numberOfActions() > 0) {
    bulkRequestBuilder.get();
  }
}
 
Example 23
Project: ecs-dashboard   File: ElasticBucketOwnerDAO.java   View source code 6 votes vote down vote up
@Override
public void insert(List<BucketOwner> bucketOwners, Date collectionTime) {
	
	if (bucketOwners != null && !bucketOwners.isEmpty()) {
		BulkRequestBuilder requestBuilder = elasticClient.prepareBulk();
		// Generate JSON for object buckets info
		for (BucketOwner bucketOwner : bucketOwners) {
			XContentBuilder objectBucketBuilder = toJsonFormat(bucketOwner, collectionTime);
			IndexRequestBuilder request = elasticClient.prepareIndex().setIndex(bucketownerIndexDayName)
					.setType(BUCKET_OWNER_INDEX_TYPE).setSource(objectBucketBuilder);
			requestBuilder.add(request);
		}

		BulkResponse bulkResponse = requestBuilder.execute().actionGet();
		int items = bulkResponse.getItems().length;
		LOGGER.info("Took " + bulkResponse.getTookInMillis() + " ms to index [" + items + "] items in ElasticSearch"
				+ "index: " + bucketownerIndexDayName + " index type: " + BUCKET_OWNER_INDEX_TYPE);

		if (bulkResponse.hasFailures()) {
			LOGGER.error("Failures occured while items in ElasticSearch " + "index: " + bucketownerIndexDayName
					+ " index type: " + BUCKET_OWNER_INDEX_TYPE);
		}
	}
}
 
Example 24
Project: elasticsearch-performance   File: Elasticsearch.java   View source code 6 votes vote down vote up
public <T> BulkResponse bulkIndex(String indexName, List<T> sources) {
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();

    sources
            .parallelStream()
            .map(src -> {
                try {
                    return Optional.of(objectMapper.writeValueAsBytes(src));
                } catch (JsonProcessingException e) {
                    return Optional.empty();
                }
            })
            .filter(Optional::isPresent)
            .forEach(src -> bulkRequestBuilder.add(new IndexRequest(indexName, indexName).source((byte[]) src.get())));

    logger.debug("Executing bulk request with size {}", sources.size());

    return bulkRequestBuilder.get();
}
 
Example 25
Project: elastic-crud   File: ElasticSearchRepository.java   View source code 6 votes vote down vote up
@Override
public List<String> deleteAllIds(final Collection<String> ids) {
  if (ids.isEmpty()) {
    return ImmutableList.of();
  }

  final BulkRequestBuilder bulk = client
    .prepareBulk()
    .setRefreshPolicy(policy.get());

  for (final String id : ids) {
    bulk.add(client.prepareDelete(index, type, id));
  }

  final BulkResponse response = bulk.execute().actionGet();

  final ImmutableList.Builder<String> builder = ImmutableList.builder();
  for (final BulkItemResponse item : response.getItems()) {
    builder.add(item.getId());
  }
  return builder.build();
}
 
Example 26
Project: elastic-crud   File: ElasticScrollingFactoryEsTest.java   View source code 6 votes vote down vote up
@Before
public void before() throws IOException {
  if(!client.admin().indices().prepareExists(INDEX).execute().actionGet().isExists()) {
    client.admin().indices().prepareCreate(INDEX).execute().actionGet();
  }
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  final BulkRequestBuilder bulk = client.prepareBulk();
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}
 
Example 27
Project: elastic-crud   File: BulkDeleteEsTest.java   View source code 6 votes vote down vote up
@Before
public void before() throws IOException {
  if(!client.admin().indices().prepareExists(INDEX).execute().actionGet().isExists()) {
    client.admin().indices().prepareCreate(INDEX).execute().actionGet();
  }
  final BulkRequestBuilder bulk = client.prepareBulk();
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}
 
Example 28
Project: elastic-crud   File: BulkIndexEsTest.java   View source code 6 votes vote down vote up
@Before
public void before() throws IOException {
  final IndicesAdminClient indices = client.admin().indices();
  if(!indices.prepareExists(INDEX).execute().actionGet().isExists()) {
    indices.prepareCreate(INDEX).execute().actionGet();
  }
  final JsonSerializer<Person> serializer = mapper.serializer(Person.class);
  final BulkRequestBuilder bulk = client.prepareBulk();
  for (int i = 0; i < SIZE; i++) {

    final String name = UUID.randomUUID().toString();
    final IndexRequest request = new IndexRequest(INDEX, TYPE);
    request.source(serializer.apply(Person.builder().id("").firstname(name).lastname(name).build()), JSON);
    bulk.add(request);
  }

  client.bulk(bulk.request()).actionGet();
  flush(INDEX);
}
 
Example 29
Project: javabase   File: UpdateMappingFieldDemo.java   View source code 6 votes vote down vote up
private static void prepareData(IndicesAdminClient indicesAdminClient) throws InterruptedException {
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    //批量添加
    for (int i = 0; i < 1000; i++) {
        ItemInfo iteminfo=new ItemInfo(i,"商品"+i, new Random().nextFloat(),new Date());
        // 当opType是Index时,如果文档id存在,更新文档,否则创建文档 当opType是Create,如果文档id存在,抛出文档存在的错误 *
        IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v1, INDEX_TYPE).setId(i+"").setSource(JSONObject.toJSONString(iteminfo)).setOpType(IndexRequest.OpType.INDEX);
        bulkRequestBuilder.add(indexRequestBuilder);
        //数据日期不一样
    }
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
     log.error("bulk index error:{}",bulkResponse.buildFailureMessage());
    } else {
        log.info("index docs : {}" , bulkResponse);
    }
}
 
Example 30
Project: zipkin   File: ElasticsearchDependenciesTest.java   View source code 6 votes vote down vote up
@VisibleForTesting void writeDependencyLinks(List<DependencyLink> links, long timestampMillis) {
  long midnight = Util.midnightUTC(timestampMillis);
  TransportClient client = ((NativeClient) storage.client()).client;
  BulkRequestBuilder request = client.prepareBulk();
  for (DependencyLink link : links) {
    request.add(client.prepareIndex(
        storage.indexNameFormatter.indexNameForTimestamp(midnight),
        ElasticsearchConstants.DEPENDENCY_LINK)
        .setId(link.parent + "|" + link.child) // Unique constraint
        .setSource(
            "parent", link.parent,
            "child", link.child,
            "callCount", link.callCount));
  }
  request.execute().actionGet();
  client.admin().indices().flush(new FlushRequest()).actionGet();
}
 
Example 31
Project: nlptab   File: DeleteOrphanedDocuments.java   View source code 6 votes vote down vote up
@Override
public void run() {
    SearchResponse searchResponse = client.prepareSearch(searchIndex)
            .setTypes("Document")
            .setQuery(QueryBuilders.boolQuery()
                    .mustNot(QueryBuilders.hasChildQuery("DocumentInSystem", QueryBuilders.matchAllQuery())))
            .setSize(9999)
            .get();

    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    for (SearchHit searchHit : searchResponse.getHits().hits()) {
        DeleteRequestBuilder delete = client.prepareDelete(searchIndex, "Document", searchHit.getId());
        bulkRequestBuilder.add(delete);
    }

    if (bulkRequestBuilder.numberOfActions() > 0) {
        bulkRequestBuilder.get();
    }
}
 
Example 32
Project: sql4es   File: ESUpdateState.java   View source code 6 votes vote down vote up
/**
 * Executes the list with requests as a bulk with maximum number of requests per bulk
 * @param requests
 * @param maxRequestsPerBulk
 * @return
 * @throws SQLException
 */
private int execute(List<?> requests, int maxRequestsPerBulk) throws SQLException{
	int result = 0;
	BulkRequestBuilder bulkReq = client.prepareBulk();
	for(Object req : requests){
		if(req instanceof IndexRequest)	bulkReq.add((IndexRequest)req);
		else if(req instanceof UpdateRequest) bulkReq.add((UpdateRequest)req);
		else if(req instanceof DeleteRequest) bulkReq.add((DeleteRequest)req);
		else if(req instanceof IndexRequestBuilder) bulkReq.add((IndexRequestBuilder)req);
		else if(req instanceof UpdateRequestBuilder) bulkReq.add((UpdateRequestBuilder)req);
		else if(req instanceof DeleteRequestBuilder) bulkReq.add((DeleteRequestBuilder)req);
		else throw new SQLException("Type "+req.getClass()+" cannot be added to a bulk request");
		
		if(bulkReq.numberOfActions() > maxRequestsPerBulk){
			result += bulkReq.get().getItems().length;
			bulkReq = client.prepareBulk();
		}
	}
	
	if(bulkReq.numberOfActions() > 0){
		result += bulkReq.get().getItems().length;
	}
	return result;
}
 
Example 33
Project: core-ng-project   File: ElasticSearchTypeImpl.java   View source code 6 votes vote down vote up
@Override
public void bulkDelete(BulkDeleteRequest request) {
    if (request.ids == null || request.ids.isEmpty()) throw Exceptions.error("request.ids must not be empty");

    StopWatch watch = new StopWatch();
    String index = request.index == null ? this.index : request.index;
    BulkRequestBuilder builder = client().prepareBulk();
    for (String id : request.ids) {
        builder.add(client().prepareDelete(index, type, id));
    }
    long esTookTime = 0;
    try {
        BulkResponse response = builder.get();
        esTookTime = response.getTook().nanos();
        if (response.hasFailures()) throw new SearchException(response.buildFailureMessage());
    } catch (ElasticsearchException e) {
        throw new SearchException(e);   // due to elastic search uses async executor to run, we have to wrap the exception to retain the original place caused the exception
    } finally {
        long elapsedTime = watch.elapsedTime();
        ActionLogContext.track("elasticsearch", elapsedTime, 0, request.ids.size());
        logger.debug("bulkDelete, index={}, type={}, size={}, esTookTime={}, elapsedTime={}", index, type, request.ids.size(), esTookTime, elapsedTime);
        checkSlowOperation(elapsedTime);
    }
}
 
Example 34
Project: baleen   File: ElasticsearchTestBase.java   View source code 6 votes vote down vote up
@Before
public void beforeTest() throws Exception{
	jCas = JCasSingleton.getJCasInstance();

	try{
		//Find all documents in index and delete the documents
		BulkRequestBuilder brb = client.prepareBulk();
		SearchHits results = client.search(new SearchRequest()).actionGet().getHits();
		for(SearchHit sh : results){
			brb.add(client.prepareDelete("baleen_index", "baleen_output", sh.getId()).request());
		}
		if(brb.numberOfActions() > 0)
			brb.get();
	}catch(IndexNotFoundException infe){
		//Index doesn't exist - ignore
	}
}
 
Example 35
Project: jobstats   File: ElasticSearchHelper.java   View source code 6 votes vote down vote up
/**
 * Effectively index the given List in ElasticSearch
 * @param positions
 * @throws IOException
 */
public static void index(List<Position> positions) throws IOException {
	Client client = buildClient();
	BulkRequestBuilder bulk = client.prepareBulk();

	Gson gson = new GsonBuilder().serializeNulls().create();

	for (Position position : positions) {

		log.info("Adding " + position.getOriginId() + " to bulk request.");
		bulk.add(client.prepareIndex("job", "doc", position.getOriginId())
				.setSource(gson.toJson(position)));
	}

	executeBulk(bulk);
}
 
Example 36
Project: elasticsearch-reindex-tool   File: IndexingComponent.java   View source code 6 votes vote down vote up
public Optional<BulkResult> indexData(ElasticDataPointer targetDataPointer, SearchHit[] hits) {
  BulkRequestBuilder bulkRequest = createBulkRequestBuilder();

  for (SearchHit hit : hits) {
    Map<String, Object> source = hit.getSource();

    IndexRequestBuilder requestBuilder = prepareIndex(targetDataPointer.getIndexName(), targetDataPointer
        .getTypeName(), hit.getId(), source, hit.getIndex());
    if (hit.getFields().get("_ttl") != null) {
      requestBuilder.setTTL(hit.getFields().get("_ttl").value());
    }
    if (hit.getFields().get("_routing") != null) {
      requestBuilder.setRouting(hit.getFields().get("_routing").value());
    }
    requestBuilder.setSource(source);
    bulkRequest.add(requestBuilder);
  }
  return executeBulk(hits.length, bulkRequest);
}
 
Example 37
Project: ElasticMangos   File: ChatDocumentProcessor.java   View source code 6 votes vote down vote up
private void processBulkRequest(BulkRequestBuilder bulkRequest, int blCount) {
    if (bulkRequest.numberOfActions() == 0) {
        return;
    }

    int currentNo = batchNo.incrementAndGet();
    logger.info("Processing bulk request {}.", currentNo);

    BulkResponse bulkResponse = bulkRequest.execute().actionGet();
    if (bulkResponse.hasFailures()) {
        logger.error("Bulk response failure! (this is bad) Request: {}, message: {}", currentNo, bulkResponse.getItems()[0].getFailureMessage());
        throw new ImportException(bulkResponse.getItems()[0].getFailureMessage());
    } else {
        logger.info("{} docs were processed by request {} with blCount of {}.", bulkRequest.numberOfActions(), currentNo, blCount);
    }
}
 
Example 38
Project: metron   File: ElasticSearchComponent.java   View source code 6 votes vote down vote up
public BulkResponse add(String indexName, String sensorType, Iterable<String> docs)
    throws IOException {
  BulkRequestBuilder bulkRequest = getClient().prepareBulk();
  for (String doc : docs) {
    IndexRequestBuilder indexRequestBuilder = getClient()
        .prepareIndex(indexName, sensorType + "_doc");

    indexRequestBuilder = indexRequestBuilder.setSource(doc);
    Map<String, Object> esDoc = JSONUtils.INSTANCE
        .load(doc, new TypeReference<Map<String, Object>>() {
        });
    indexRequestBuilder.setId((String) esDoc.get(Constants.GUID));
    Object ts = esDoc.get("timestamp");
    if (ts != null) {
      indexRequestBuilder = indexRequestBuilder.setTimestamp(ts.toString());
    }
    bulkRequest.add(indexRequestBuilder);
  }

  BulkResponse response = bulkRequest.execute().actionGet();
  if (response.hasFailures()) {
    throw new IOException(response.buildFailureMessage());
  }
  return response;
}
 
Example 39
Project: miabis-converter   File: ElasticSearchSampleWriter.java   View source code 6 votes vote down vote up
@Override
public void write(List<? extends Sample> items) throws Exception {
	
	BulkRequestBuilder bulkRequest = client.prepareBulk();
	
	for(Sample sample : items){
		byte[] jsonSample = mapper.writeValueAsBytes(sample);
		
		if(delete){
			bulkRequest.add(client.prepareDelete(index, "sample", sample.getId()));
		}else{
			bulkRequest.add(client.prepareIndex(index, "sample", sample.getId()).setSource(jsonSample));
		}
	}
	
	BulkResponse res = bulkRequest.execute().actionGet();
}
 
Example 40
Project: elasticsearch-akka   File: IndexProductDataServiceImpl.java   View source code 6 votes vote down vote up
protected BulkResponse processBulkRequests(final List<IndexRequestBuilder> requests)
{
    if (requests.size() > 0)
    {
        final BulkRequestBuilder bulkRequest = searchClientService.getClient().prepareBulk();
        for (final IndexRequestBuilder indexRequestBuilder : requests)
        {
            bulkRequest.add(indexRequestBuilder);
        }
        logger.debug("Executing bulk index request for size:" + requests.size());
        final BulkResponse bulkResponse = bulkRequest.execute().actionGet();
        logger.debug("Bulk operation data index response total items is:" + bulkResponse.getItems().length);
        if (bulkResponse.hasFailures())
        {
            // process failures by iterating through each bulk response item
            logger.error("bulk operation indexing has failures:" + bulkResponse.buildFailureMessage());
        }
        return bulkResponse;
    }
    else
    {
        logger.debug("Executing bulk index request for size: 0");
        return null;
    }
}