org.elasticsearch.action.index.IndexRequest Java Examples

private void indexObject(final String index, final String docType, final String docId, final Object doc) {

        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(doc);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert {} '{}' to byte string", docType, docId);
            return;
        }

        IndexRequest request = new IndexRequest(index, docType, docId);
        request.source(docBytes, XContentType.JSON);

        if(bulkRequests.get(docType) == null) {
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }

        bulkRequests.get(docType).getBulkRequest().add(request);
        if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) {
            indexBulkRequest(docType);
        }
    }
 

protected Pair<BulkRequest, FutureCallbackHolder> prepareBatch(Batch<Object> batch, WriteCallback callback) {
  BulkRequest bulkRequest = new BulkRequest();
  final StringBuilder stringBuilder = new StringBuilder();
  for (Object record : batch.getRecords()) {
    try {
      byte[] serializedBytes = this.serializer.serializeToJson(record);
      log.debug("serialized record: {}", serializedBytes);
      IndexRequest indexRequest = new IndexRequest(this.indexName, this.indexType)
          .source(serializedBytes, 0, serializedBytes.length, XContentType.JSON);
      if (this.idMappingEnabled) {
        String id = this.typeMapper.getValue(this.idFieldName, record);
        indexRequest.id(id);
        stringBuilder.append(";").append(id);
      }
      bulkRequest.add(indexRequest);
    }
    catch (Exception e) {
      log.error("Encountered exception {}", e);
    }
  }
  FutureCallbackHolder futureCallbackHolder = new FutureCallbackHolder(callback,
      exception -> log.error("Batch: {} failed on ids; {} with exception {}", batch.getId(),
          stringBuilder.toString(), exception),
      this.malformedDocPolicy);
  return new Pair(bulkRequest, futureCallbackHolder);
}
 

@Override
public boolean indexData(IndexMetadata indexMetadata, String data) {
  IndexRequest indexRequest = new IndexRequest(indexMetadata.getName(), TYPE);
  if (indexMetadata.getId() != null && !indexMetadata.getId().isEmpty()) {
    indexRequest.id(indexMetadata.getId());
  }
  indexRequest.source(data, XContentType.JSON);
  indexRequest.routing(indexMetadata.getRouting());

  try {
    IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
    return indexResponse.status().equals(RestStatus.CREATED) || indexResponse.status().equals(RestStatus.OK);
  } catch (IOException e) {
    log.error("Could not index '#{}' to index '{}'.", indexMetadata.getRouting(), indexMetadata.getName(), e);
  }
  return false;
}
 

@Override
public IndexResponse persist(Map<String, Object> dataMap) throws ElasticsearchGenerationException, IOException {
    if (!settings.isLoggingEnabled()) {
        return null;
    }
    if (adminHandler.getElasticsearchClient() == null) {
        throw new NullPointerException("Client is not initialized. Data will not be persisted.");
    }

    dataMap.put(ServiceEventDataMapping.TIMESTAMP_FIELD.getName(), DateTime.now(DateTimeZone.UTC));
    dataMap.put(ServiceEventDataMapping.UUID_FIELD.getName(), settings.getUuid());
    logger.debug("Persisting {}", dataMap);
    IndexResponse response = adminHandler.getElasticsearchClient().index(
            new IndexRequest(settings.getIndexId()).type(settings.getTypeId()).source(dataMap),
            RequestOptions.DEFAULT);
    return response;
}
 

private void putTBRDocs() throws IOException, JSONException
{
    File[] files = getFiles("docs/tbr");
    for (File file : files)
    {
        String content = readFile(file);
        JSONObject jsonObject = new JSONObject(content);
        ObjectMapper mapper = new ObjectMapper();
        Map map = mapper.readValue(content, new TypeReference<Map<String, Object>>()
        {
        });
        rhlclient.index(new IndexRequest()
                .index(properties.getProperty("es.tbr.index"))
                .type(properties.getProperty("es.tbr.type"))
                .id(jsonObject.getString("uckey")).source(map));
    }
}
 

@Test
public void putModelCheckpoint_getIndexRequest() {
    checkpointDao.putModelCheckpoint(modelId, model);

    ArgumentCaptor<IndexRequest> indexRequestCaptor = ArgumentCaptor.forClass(IndexRequest.class);
    verify(clientUtil)
        .timedRequest(
            indexRequestCaptor.capture(),
            anyObject(),
            Matchers.<BiConsumer<IndexRequest, ActionListener<IndexResponse>>>anyObject()
        );
    IndexRequest indexRequest = indexRequestCaptor.getValue();
    assertEquals(indexName, indexRequest.index());
    assertEquals(CheckpointDao.DOC_TYPE, indexRequest.type());
    assertEquals(modelId, indexRequest.id());
    Set<String> expectedSourceKeys = new HashSet<String>(Arrays.asList(CheckpointDao.FIELD_MODEL, CheckpointDao.TIMESTAMP));
    assertEquals(expectedSourceKeys, indexRequest.sourceAsMap().keySet());
    assertEquals(model, indexRequest.sourceAsMap().get(CheckpointDao.FIELD_MODEL));
    assertNotNull(indexRequest.sourceAsMap().get(CheckpointDao.TIMESTAMP));
}
 

private void indexDataPoint(TaskInfo taskInfo, Date timestamp, String name,
        double value) {
    try {
        XContentBuilder builder = jsonBuilder().startObject();
        builder.field("stormId", stormID);
        builder.field("srcComponentId", taskInfo.srcComponentId);
        builder.field("srcTaskId", taskInfo.srcTaskId);
        builder.field("srcWorkerHost", taskInfo.srcWorkerHost);
        builder.field("srcWorkerPort", taskInfo.srcWorkerPort);
        builder.field("name", name);
        builder.field("value", value);
        builder.field("timestamp", timestamp);
        builder.endObject();

        IndexRequest indexRequest = new IndexRequest(
                getIndexName(timestamp)).source(builder);
        connection.getProcessor().add(indexRequest);
    } catch (Exception e) {
        LOG.error("problem when building request for ES", e);
    }
}
 

private void index(RestHighLevelClient client, String indexName) throws IOException {
    XContentBuilder builder = XContentFactory.jsonBuilder();
    builder.startObject();
    {
        builder.field("author", "Marker");
        builder.field("title", "Java programing.");
    }
    builder.endObject();
    IndexRequest indexRequest = new IndexRequest(indexName, "_doc", "1").source(builder);

    IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
    if (indexResponse.status().getStatus() >= 400) {
        String message = "elasticsearch index data fail.";
        logger.error(message);
        throw new RuntimeException(message);
    }
}
 

/**
 * Prepare a Runnable to send an index request to store the element, invalidates the cache on success
 */
private void store(FeatureStoreRequest request, Task task, ActionListener<FeatureStoreResponse> listener) {

    try {
        Optional<ClearCachesNodesRequest> clearCachesNodesRequest = buildClearCache(request);
        IndexRequest indexRequest = buildIndexRequest(task, request);
        client.execute(IndexAction.INSTANCE, indexRequest, wrap(
                (r) -> {
                    // Run and forget, log only if something bad happens
                    // but don't wait for the action to be done nor set the parent task.
                    clearCachesNodesRequest.ifPresent((req) -> clearCachesAction.execute(req, wrap(
                            (r2) -> {
                            },
                            (e) -> logger.error("Failed to clear cache", e))));
                    listener.onResponse(new FeatureStoreResponse(r));
                },
                listener::onFailure));
    } catch (IOException ioe) {
        listener.onFailure(ioe);
    }
}
 

@Test
public void testSavingAdResult() throws IOException {
    setUpSavingAnomalyResultIndex(false);
    doAnswer(invocation -> {
        Object[] args = invocation.getArguments();
        assertTrue(String.format("The size of args is %d.  Its content is %s", args.length, Arrays.toString(args)), args.length >= 2);
        IndexRequest request = invocation.getArgument(0);
        ActionListener<IndexResponse> listener = invocation.getArgument(1);
        assertTrue(request != null && listener != null);
        listener.onResponse(mock(IndexResponse.class));
        return null;
    }).when(client).index(any(IndexRequest.class), ArgumentMatchers.<ActionListener<IndexResponse>>any());
    AnomalyResultHandler handler = new AnomalyResultHandler(
        client,
        settings,
        clusterService,
        indexNameResolver,
        anomalyDetectionIndices,
        threadPool
    );
    handler.indexAnomalyResult(TestHelpers.randomAnomalyDetectResult());
    assertEquals(1, testAppender.countMessage((AnomalyResultHandler.SUCCESS_SAVING_MSG)));
}
 

@Override
protected void populateData(TransportClient tc) {

    tc.index(new IndexRequest("deals").type("deals").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"amount\": 10}", XContentType.JSON)).actionGet();
    tc.index(new IndexRequest("deals").type("deals").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"amount\": 1500}", XContentType.JSON)).actionGet();

    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    System.out.println("q");
    System.out.println(Strings.toString(tc.search(new SearchRequest().indices(".opendistro_security")).actionGet()));
    tc.search(new SearchRequest().indices("deals")).actionGet();
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeString(index);
    timeout.writeTo(out);
    out.writeLong(ingestId);
    shardId.writeTo(out);
    out.writeVInt(actionRequests.size());
    for (ActionRequest<?> actionRequest : actionRequests) {
        if (actionRequest == null) {
            out.writeBoolean(false);
            continue;
        }
        out.writeBoolean(true);
        if (actionRequest instanceof IndexRequest) {
            out.writeBoolean(true);
        } else if (actionRequest instanceof DeleteRequest) {
            out.writeBoolean(false);
        } else {
            throw new ElasticsearchException("action request not supported: " + actionRequest.getClass().getName());
        }
        actionRequest.writeTo(out);
    }
}
 

private static void prepareData(IndicesAdminClient indicesAdminClient) throws InterruptedException {
    BulkRequestBuilder bulkRequestBuilder = client.prepareBulk();
    //批量添加
    for (int i = 0; i < 1000; i++) {
        ItemInfo iteminfo=new ItemInfo(i,"商品"+i, new Random().nextFloat(),new Date());
        // 当opType是Index时,如果文档id存在,更新文档,否则创建文档 当opType是Create,如果文档id存在,抛出文档存在的错误 *
        IndexRequestBuilder indexRequestBuilder = client.prepareIndex(INDEX_NAME_v1, INDEX_TYPE).setId(i+"").setSource(JSONObject.toJSONString(iteminfo)).setOpType(IndexRequest.OpType.INDEX);
        bulkRequestBuilder.add(indexRequestBuilder);
        //数据日期不一样
    }
    BulkResponse bulkResponse = bulkRequestBuilder.execute().actionGet();
    if (bulkResponse.hasFailures()) {
     log.error("bulk index error:{}",bulkResponse.buildFailureMessage());
    } else {
        log.info("index docs : {}" , bulkResponse);
    }
}
 

@Override
public void writeTo(StreamOutput out) throws IOException {
    super.writeTo(out);
    out.writeByte(consistencyLevel.id());
    out.writeVInt(requests.size());
    for (ActionRequest request : requests) {
        if (request instanceof IndexRequest) {
            out.writeByte((byte) 0);
        } else if (request instanceof DeleteRequest) {
            out.writeByte((byte) 1);
        } else if (request instanceof UpdateRequest) {
            out.writeByte((byte) 2);
        }
        request.writeTo(out);
    }
    out.writeBoolean(refresh);
    timeout.writeTo(out);
}
 

/**
 * bulkProcessor 批量增加
 * @Author lihaodong
 * @Description
 * @Date 20:17 2018/12/21
 * @Param [indexName, type, bookList]
 * @return void
 **/
public static void bulkProcessorAdd(String indexName, String type, List<Book> bookList) {
    bookList.stream().parallel().forEach(
            book -> {
                try {
                    bulk.add(new IndexRequest(indexName, type, book.getId()).source(XContentFactory.jsonBuilder()
                            .startObject()
                            .field("id", book.getId())
                            .field("name", book.getName())
                            .field("creatDate", book.getCreatDate())
                            .field("price", book.getPrice())
                            .field("message", book.getMessage())
                            .endObject()));
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
    );
}
 

@Test
public void bulk() throws InterruptedException {
    BulkProcessor bulkProcessor = client.createBulkProcessor(2000, 10, 2);

    Map<String, String> source = new HashMap<>();
    source.put("column1", "value1");
    source.put("column2", "value2");

    for (int i = 0; i < 100; i++) {
        IndexRequest indexRequest = new IndexRequest("bulk_insert_test", "type", String.valueOf(i));
        indexRequest.source(source);
        bulkProcessor.add(indexRequest);
    }

    bulkProcessor.flush();
    bulkProcessor.awaitClose(2, TimeUnit.SECONDS);
}
 

private IndexRequest prepareRequest(final TikaDocument document, final TikaDocument parent, TikaDocument root, final int level) throws IOException {
    IndexRequest req = new IndexRequest(indexName, esCfg.indexType, document.getId());
    Map<String, Object> jsonDocument = getDocumentMap(document);

    if (parent == null && isDuplicate(document.getId())) {
        IndexRequest indexRequest = new IndexRequest(indexName, esCfg.indexType, Entity.HASHER.hash(document.getPath()));
        indexRequest.source(getDuplicateMap(document));
        indexRequest.setRefreshPolicy(esCfg.refreshPolicy);
        return indexRequest;
    }

    if (parent != null) {
        jsonDocument.put(DEFAULT_PARENT_DOC_FIELD, parent.getId());
        jsonDocument.put("rootDocument", root.getId());
        req.routing(root.getId());
    }
    jsonDocument.put("extractionLevel", level);
    req = req.source(jsonDocument);
    req.setRefreshPolicy(esCfg.refreshPolicy);
    return req;
}
 

private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) {
	Map<String, Object> json = new HashMap<>();
	json.put("data", element);

	String index;
	String type;

	if (element.startsWith("message #15")) {
		index = ":intentional invalid index:";
		type = ":intentional invalid type:";
	} else {
		index = parameterTool.getRequired("index");
		type = parameterTool.getRequired("type");
	}

	return Requests.indexRequest()
		.index(index)
		.type(type)
		.id(element)
		.source(json);
}
 

@Override
public IndexRequest getIndexRequest(OutgoingMessageEnvelope envelope) {
  IndexRequest indexRequest = getRequest(envelope);

  getId(envelope).ifPresent(indexRequest::id);
  getRoutingKey(envelope).ifPresent(indexRequest::routing);
  getVersion(envelope).ifPresent(indexRequest::version);
  getVersionType(envelope).ifPresent(indexRequest::versionType);

  setSource(envelope, indexRequest);

  return indexRequest;
}
 

@Override
public Optional<EventId> saveData(EventDataInfo eventDataInfo) throws IOException {
    IndexRequest indexRequest = new IndexRequest(INDEX_NAME);
    indexRequest.source(Utils.createContent(eventDataInfo));
    IndexResponse indexResponse = client.index(indexRequest, RequestOptions.DEFAULT);
    if (RestStatus.CREATED.equals(indexResponse.status())) {
        return Optional.of(new EventId(indexResponse.getId()));
    }
    return Optional.empty();
}
 

protected UpdateResponse updateOrInsert(String id, byte[] updateDoc, byte[] insertDoc)
    throws Exception {
  IndexRequest
      indexRequest =
      new IndexRequest(getIndexName(), getDocTypeName(), id).source(insertDoc);

  UpdateRequest
      updateRequest =
      new UpdateRequest(getIndexName(), getDocTypeName(), id).doc(updateDoc).upsert(indexRequest);
  return esClient.update(updateRequest).actionGet();
}
 

private void createDocument() throws IOException, ExecutionException, InterruptedException {
    IndexResponse ir = doIndex(new IndexRequest(INDEX, DOC_TYPE, DOC_ID).source(
        jsonBuilder()
            .startObject()
            .field(FOO, BAR)
            .endObject()
    ).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE));
    assertThat(ir.status().getStatus()).isEqualTo(201);
}
 

private IndexRequest buildIndexRequest(Task parentTask, FeatureStoreRequest request) throws IOException {
    StorableElement elt = request.getStorableElement();

    IndexRequest indexRequest = client.prepareIndex(request.getStore(), IndexFeatureStore.ES_TYPE, elt.id())
            .setCreate(request.getAction() == FeatureStoreRequest.Action.CREATE)
            .setRouting(request.getRouting())
            .setSource(IndexFeatureStore.toSource(elt))
            .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE)
            .request();
    indexRequest.setParentTask(clusterService.localNode().getId(), parentTask.getId());
    return indexRequest;
}
 

private void flush() {
    MultiGetResponse multiGetItemResponse = request.execute().actionGet();

    for(MultiGetItemResponse response : multiGetItemResponse.getResponses()) {
        if (response.isFailed()) {
            logger.error("Get failed: " + response.getFailure().getMessage());
        }

        Metric metric = request.metrics.get(response.getId());
        if (response.isFailed() || !response.getResponse().isExists()) {
            final String[] parts = metric.getPath().split("\\.");
            final StringBuilder sb = new StringBuilder();

            for (int i = 0; i < parts.length; i++) {
                if (sb.toString().length() > 0) {
                    sb.append(".");
                }
                sb.append(parts[i]);
                try {
                    bulkProcessor.add(new IndexRequest(index, type, metric.getTenant() + "_" + sb.toString()).source(
                            XContentFactory.jsonBuilder().startObject()
                                    .field("tenant", metric.getTenant())
                                    .field("path", sb.toString())
                                    .field("depth", (i + 1))
                                    .field("leaf", (i == parts.length - 1))
                                    .endObject()
                    ));
                } catch (IOException e) {
                    logger.error(e);
                }

            }

        }

    }

    request = new MetricMultiGetRequestBuilder(client, index, type);
}
 

public IndexResponse index(IndexRequest indexRequest, String json) {
    try {
        indexRequest.source(json, XContentType.JSON);
        return legacyElasticSearchClient.index(indexRequest, RequestOptions.DEFAULT);
    } catch (IOException e) {
        log.error(String.format("Indexing mapping %s failed", json), e);
        throw new RuntimeException(e);
    }
}
 

private IndexRequest createIndexRequest(String index, String type, String id, Map<String, Object> json, String parent, String root) {
    IndexRequest req = new IndexRequest(index, esCfg.indexType, id);

    setJoinFields(json, type, parent, root);
    req = req.source(json);
    return (parent != null) ? req.routing(root) : req;
}
 

public IndexRequest createIndexRequest(Tuple2<Integer, String> element) {
	Map<String, Object> document = new HashMap<>();
	document.put(DATA_FIELD_NAME, element.f1);

	try {
		return new IndexRequest(index, TYPE_NAME, element.f0.toString()).source(contentBuilderProvider.getBuilder().map(document));
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

public IngestRequest add(ActionRequest<?> request) {
    if (request instanceof IndexRequest) {
        add((IndexRequest) request);
    } else if (request instanceof DeleteRequest) {
        add((DeleteRequest) request);
    } else {
        throw new IllegalArgumentException("no support for request [" + request + "]");
    }
    return this;
}
 

public String upsert(String index,String type,String id,Object json){
	try {
		if(client==null){
			init();
		}
		IndexRequest indexRequest = new IndexRequest(index, type, id).source(JSON.parseObject(JSON.toJSONString(json)),XContentType.JSON);
		UpdateRequest updateRequest = new UpdateRequest(index, type, id).doc(JSON.parseObject(JSON.toJSONString(json)),XContentType.JSON).upsert(indexRequest);              
		UpdateResponse result = client.update(updateRequest).get();
		return result.toString();
	}catch (Exception e) {
		// TODO Auto-generated catch block
		e.printStackTrace();
	}
	return null;
}
 

/**
 * Add multiple {@link ActionRequest} to the indexer to prepare for sending requests to Elasticsearch.
 *
 * @param actionRequests The multiple {@link ActionRequest} to add.
 * @deprecated use the {@link DeleteRequest}, {@link IndexRequest} or {@link UpdateRequest}
 */
@Deprecated
default void add(ActionRequest... actionRequests) {
	for (ActionRequest actionRequest : actionRequests) {
		if (actionRequest instanceof IndexRequest) {
			add((IndexRequest) actionRequest);
		} else if (actionRequest instanceof DeleteRequest) {
			add((DeleteRequest) actionRequest);
		} else if (actionRequest instanceof UpdateRequest) {
			add((UpdateRequest) actionRequest);
		} else {
			throw new IllegalArgumentException("RequestIndexer only supports Index, Delete and Update requests");
		}
	}
}