org.elasticsearch.common.xcontent.XContentType Java Examples

The following examples show how to use org.elasticsearch.common.xcontent.XContentType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: conductor   Author: Netflix   File: ElasticSearchRestDAOV5.java    License: Apache License 2.0 7 votes vote down vote up
private void indexObject(final String index, final String docType, final String docId, final Object doc) {

        byte[] docBytes;
        try {
            docBytes = objectMapper.writeValueAsBytes(doc);
        } catch (JsonProcessingException e) {
            logger.error("Failed to convert {} '{}' to byte string", docType, docId);
            return;
        }

        IndexRequest request = new IndexRequest(index, docType, docId);
        request.source(docBytes, XContentType.JSON);

        if(bulkRequests.get(docType) == null) {
            bulkRequests.put(docType, new BulkRequests(System.currentTimeMillis(), new BulkRequest()));
        }

        bulkRequests.get(docType).getBulkRequest().add(request);
        if (bulkRequests.get(docType).getBulkRequest().numberOfActions() >= this.indexBatchSize) {
            indexBulkRequest(docType);
        }
    }
 
Example #2
Source Project: crate   Author: crate   File: XContentTestUtilsTests.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public void testInsertIntoXContent() throws IOException {
    XContentBuilder builder = JsonXContent.contentBuilder();
    builder.startObject();
    builder.endObject();
    builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), BytesReference.bytes(builder),
            Collections.singletonList(""), () -> "inn.er1", () -> new HashMap<>());
    builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), BytesReference.bytes(builder),
            Collections.singletonList(""), () -> "field1", () -> "value1");
    builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), BytesReference.bytes(builder),
            Collections.singletonList("inn\\.er1"), () -> "inner2", () -> new HashMap<>());
    builder = XContentTestUtils.insertIntoXContent(XContentType.JSON.xContent(), BytesReference.bytes(builder),
            Collections.singletonList("inn\\.er1"), () -> "field2", () -> "value2");
    try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY,
        DeprecationHandler.THROW_UNSUPPORTED_OPERATION, BytesReference.bytes(builder), builder.contentType())) {
        Map<String, Object> map = parser.map();
        assertEquals(2, map.size());
        assertEquals("value1", map.get("field1"));
        assertThat(map.get("inn.er1"), instanceOf(Map.class));
        Map<String, Object> innerMap = (Map<String, Object>) map.get("inn.er1");
        assertEquals(2, innerMap.size());
        assertEquals("value2", innerMap.get("field2"));
        assertThat(innerMap.get("inner2"), instanceOf(Map.class));
        assertEquals(0, ((Map<String, Object>) innerMap.get("inner2")).size());
    }
}
 
Example #3
Source Project: sql4es   Author: Anchormen   File: Sql4EsBase.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the index with optionally the mapping and a number of docs
 * @param index
 * @param type
 * @param withMapping
 * @param nrDocs
 * @throws IOException
 */
protected void createIndexTypeWithDocs(String index, String type, boolean withMapping, int nrDocs) throws IOException{
	if(withMapping){
		String mapping = AccessController.doPrivileged(new PrivilegedAction<String>(){
			@Override
			public String run() {
				try {
					return new String(Files.readAllBytes(Paths.get("src/test/resources/TestDocumentMapping.json")));
				} catch (IOException e) {
					return null;
				}
			}
		});
		if(mapping == null) throw new IOException("Unable to read TestDocumentMapping.json");
		client().admin().indices().prepareCreate(index).addMapping(type, mapping, XContentType.JSON).get(); //.execute().actionGet();
	}else{
		createIndex(index);
	}
	if(nrDocs > 0) addDocs(index, type, nrDocs);
	refresh();
}
 
Example #4
Source Project: zentity   Author: zentity-io   File: Job.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Submit a search query to Elasticsearch.
 *
 * @param indexName The name of the index to search.
 * @param query     The query to search.
 * @return The search response returned by Elasticsearch.
 * @throws IOException
 */
private SearchResponse search(String indexName, String query) throws IOException {
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    SearchModule searchModule = new SearchModule(Settings.EMPTY, false, Collections.emptyList());
    try (XContentParser parser = XContentFactory.xContent(XContentType.JSON).createParser(new NamedXContentRegistry(searchModule
            .getNamedXContents()), DeprecationHandler.THROW_UNSUPPORTED_OPERATION, query)) {
        searchSourceBuilder.parseXContent(parser);
    }
    SearchRequestBuilder searchRequestBuilder = new SearchRequestBuilder(client, SearchAction.INSTANCE);
    searchRequestBuilder.setIndices(indexName).setSource(searchSourceBuilder);
    if (this.searchAllowPartialSearchResults != null)
        searchRequestBuilder.setAllowPartialSearchResults(this.searchAllowPartialSearchResults);
    if (this.searchBatchedReduceSize != null)
        searchRequestBuilder.setBatchedReduceSize(this.searchBatchedReduceSize);
    if (this.searchMaxConcurrentShardRequests != null)
        searchRequestBuilder.setMaxConcurrentShardRequests(this.searchMaxConcurrentShardRequests);
    if (this.searchPreFilterShardSize != null)
        searchRequestBuilder.setPreFilterShardSize(this.searchPreFilterShardSize);
    if (this.searchPreference != null)
        searchRequestBuilder.setPreference(this.searchPreference);
    if (this.searchRequestCache != null)
        searchRequestBuilder.setRequestCache(this.searchRequestCache);
    if (this.maxTimePerQuery != null)
        searchRequestBuilder.setTimeout(TimeValue.parseTimeValue(this.maxTimePerQuery, "timeout"));
    return searchRequestBuilder.execute().actionGet();
}
 
Example #5
Source Project: elasticsearch-sql   Author: NLPchina   File: MainTestSuite.java    License: Apache License 2.0 6 votes vote down vote up
private static void prepareAccountsIndex() {
    String dataMapping = "{  \"account\": {" +
            " \"properties\": {\n" +
            "          \"gender\": {\n" +
            "            \"type\": \"text\",\n" +
            "            \"fielddata\": true\n" +
            "          }," +
            "          \"address\": {\n" +
            "            \"type\": \"text\",\n" +
            "            \"fielddata\": true\n" +
            "          }," +
            "          \"state\": {\n" +
            "            \"type\": \"text\",\n" +
            "            \"fielddata\": true\n" +
            "          }" +
            "       }"+
            "   }" +
            "}";
    client.admin().indices().preparePutMapping(TEST_INDEX_ACCOUNT).setType("account").setSource(dataMapping, XContentType.JSON).execute().actionGet();
}
 
Example #6
Source Project: elasticsearch-rest-command   Author: huangchen007   File: CsvXContentGenerator.java    License: The Unlicense 6 votes vote down vote up
@Override
public final void writeRawField(String fieldName, BytesReference content, OutputStream bos) throws IOException {
    XContentType contentType = XContentFactory.xContentType(content);
    if (contentType != null) {
        writeObjectRaw(fieldName, content, bos);
    } else {
        writeFieldName(fieldName);
        // we could potentially optimize this to not rely on exception logic...
        String sValue = content.toUtf8();
        try {
            writeNumber(Long.parseLong(sValue));
        } catch (NumberFormatException e) {
            try {
                writeNumber(Double.parseDouble(sValue));
            } catch (NumberFormatException e1) {
                writeString(sValue);
            }
        }
    }
}
 
Example #7
Source Project: crate   Author: crate   File: GeneratedColsFromRawInsertSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Map<String, Object> generateSourceAndCheckConstraints(Object[] values) {
    String rawSource = (String) values[0];
    Map<String, Object> source = XContentHelper.toMap(new BytesArray(rawSource), XContentType.JSON);
    mixinDefaults(source, defaults);
    for (int i = 0; i < expressions.size(); i++) {
        expressions.get(i).setNextRow(source);
    }
    for (Map.Entry<Reference, Input<?>> entry : generatedCols.entrySet()) {
        var reference = entry.getKey();
        var value = entry.getValue().value();
        var valueForInsert = reference
            .valueType()
            .valueForInsert(value);
        source.putIfAbsent(reference.column().fqn(), valueForInsert);
    }
    return source;
}
 
Example #8
Source Project: flink-learning   Author: zhisheng17   File: LogSink2ES.java    License: Apache License 2.0 6 votes vote down vote up
public static void sink2es(SingleOutputStreamOperator<LogEvent> logDataStream, ParameterTool parameterTool) {
    List<HttpHost> esAddresses;
    try {
         esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    } catch (MalformedURLException e) {
        log.error("get es address has an error", e);
        return;
    }
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, logDataStream,
            (LogEvent logEvent, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index("zhisheng_log")
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(logEvent), XContentType.JSON));
            },
            parameterTool);
}
 
Example #9
Source Project: conductor   Author: Netflix   File: ElasticSearchDAOV6.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void addMessage(String queue, Message message) {
    try {
        long startTime = Instant.now().toEpochMilli();
        Map<String, Object> doc = new HashMap<>();
        doc.put("messageId", message.getId());
        doc.put("payload", message.getPayload());
        doc.put("queue", queue);
        doc.put("created", System.currentTimeMillis());

        String docType = StringUtils.isBlank(docTypeOverride) ? MSG_DOC_TYPE : docTypeOverride;
        UpdateRequest req = new UpdateRequest(messageIndexName, docType, message.getId());
        req.doc(doc, XContentType.JSON);
        req.upsert(doc, XContentType.JSON);
        indexObject(req, MSG_DOC_TYPE);
        long endTime = Instant.now().toEpochMilli();
        LOGGER.debug("Time taken {} for  indexing message: {}", endTime - startTime, message.getId());
        Monitors.recordESIndexTime("add_message", MSG_DOC_TYPE, endTime - startTime);
    } catch (Exception e) {
        LOGGER.error("Failed to index message: {}", message.getId(), e);
    }
}
 
Example #10
Source Project: deprecated-security-advanced-modules   Author: opendistro-for-elasticsearch   File: DlsTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void populateData(TransportClient tc) {

    tc.index(new IndexRequest("deals").type("deals").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"amount\": 10}", XContentType.JSON)).actionGet();
    tc.index(new IndexRequest("deals").type("deals").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
            .source("{\"amount\": 1500}", XContentType.JSON)).actionGet();

    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    System.out.println("q");
    System.out.println(Strings.toString(tc.search(new SearchRequest().indices(".opendistro_security")).actionGet()));
    tc.search(new SearchRequest().indices("deals")).actionGet();
}
 
Example #11
Source Project: elasticsearch-sql   Author: NLPchina   File: JoinRequestBuilder.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public String explain() {
    try {
        XContentBuilder firstBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
        firstTable.getRequestBuilder().request().source().toXContent(firstBuilder, ToXContent.EMPTY_PARAMS);

        XContentBuilder secondBuilder = XContentFactory.contentBuilder(XContentType.JSON).prettyPrint();
        secondTable.getRequestBuilder().request().source().toXContent(secondBuilder, ToXContent.EMPTY_PARAMS);
        String explained = String.format(" first query:\n%s\n second query:\n%s", BytesReference.bytes(firstBuilder).utf8ToString(), BytesReference.bytes(secondBuilder).utf8ToString());

        return explained;
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}
 
Example #12
Source Project: vertx-elasticsearch-service   Author: hubrick   File: DefaultElasticSearchAdminService.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void putTemplate(String name, JsonObject source, TemplateOptions options, Handler<AsyncResult<Void>> resultHandler) {

    final PutIndexTemplateRequestBuilder builder = new PutIndexTemplateRequestBuilder(service.getClient(), PutIndexTemplateAction.INSTANCE, name)
            .setSource(source.encode().getBytes(Charsets.UTF_8), XContentType.JSON);

    builder.execute(new ActionListener<PutIndexTemplateResponse>() {
        @Override
        public void onResponse(PutIndexTemplateResponse putIndexTemplateResponse) {
            resultHandler.handle(Future.succeededFuture());
        }

        @Override
        public void onFailure(Exception e) {
            resultHandler.handle(Future.failedFuture(e));
        }
    });
}
 
Example #13
Source Project: mongolastic   Author: ozlerhakan   File: ElasticBulkService.java    License: MIT License 6 votes vote down vote up
@Override
public void proceed(List content) {
    try {
        logger.info("Transferring data began to elasticsearch.");
        final String indexName = config.getMisc().getDindex().getAs();
        final String typeName = config.getMisc().getCtype().getAs();
        for (Object o : content) {
            Document doc = (Document) o;
            Object id = doc.get("_id");
            IndexRequest indexRequest = new IndexRequest(indexName, typeName, String.valueOf(id));
            doc.remove("_id");
            indexRequest.source(doc.toJson(encoder), XContentType.JSON);
            bulkProcessor.add(indexRequest);
        }
    } catch (Exception ex) {
        logger.debug(ex.getMessage(), ex);
    }
}
 
Example #14
Source Project: crate   Author: crate   File: ElasticsearchAssertions.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Asserts that the provided {@link BytesReference}s created through
 * {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} hold the same content.
 * The comparison is done by parsing both into a map and comparing those two, so that keys ordering doesn't matter.
 * Also binary values (byte[]) are properly compared through arrays comparisons.
 */
public static void assertToXContentEquivalent(BytesReference expected, BytesReference actual, XContentType xContentType)
        throws IOException {
    //we tried comparing byte per byte, but that didn't fly for a couple of reasons:
    //1) whenever anything goes through a map while parsing, ordering is not preserved, which is perfectly ok
    //2) Jackson SMILE parser parses floats as double, which then get printed out as double (with double precision)
    //Note that byte[] holding binary values need special treatment as they need to be properly compared item per item.
    Map<String, Object> actualMap = null;
    Map<String, Object> expectedMap = null;
    try (XContentParser actualParser = xContentType.xContent()
            .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, actual.streamInput())) {
        actualMap = actualParser.map();
        try (XContentParser expectedParser = xContentType.xContent()
                .createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, expected.streamInput())) {
            expectedMap = expectedParser.map();
            try {
                assertMapEquals(expectedMap, actualMap);
            } catch (AssertionError error) {
                NotEqualMessageBuilder message = new NotEqualMessageBuilder();
                message.compareMaps(actualMap, expectedMap);
                throw new AssertionError("Error when comparing xContent.\n" + message.toString(), error);
            }
        }
    }
}
 
Example #15
Source Project: siren-join   Author: sirensolutions   File: BaseTransportCoordinateSearchAction.java    License: GNU Affero General Public License v3.0 6 votes vote down vote up
protected Tuple<XContentType, Map<String, Object>> parseSource(BytesReference source) {
  // nothing to parse...
  if (source == null || source.length() == 0) {
    return null;
  }

  try {
    Tuple<XContentType, Map<String, Object>> parsedSource = XContentHelper.convertToMap(source, false);
    logger.debug("{}: Parsed source: {}", Thread.currentThread().getName(), parsedSource);
    return parsedSource;
  }
  catch (Throwable e) {
      String sSource = "_na_";
      try {
          sSource = XContentHelper.convertToJson(source, false);
      }
      catch (Throwable e1) { /* ignore  */ }
      throw new ElasticsearchParseException("Failed to parse source [" + sSource + "]", e);
  }
}
 
Example #16
Source Project: search-spring-boot-starter   Author: niuzhiweimr   File: ESNestedSearchService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * 获取新增请求
 *
 * @param esObject 更新请求参数
 * @return UpdateRequestBuilder 更新请求
 */
private IndexRequestBuilder getIndexRequest(SaveESObject esObject) {
    String dataId = getId(esObject.getUkMap());

    byte[] dataBytes = null;
    try {
        dataBytes = OBJECT_MAPPER.writeValueAsBytes(esObject.getDataMap());
    } catch (JsonProcessingException e) {
        // never hapened
        SearchLogger.error("", e);
    }

    IndexRequestBuilder indexRequestBuilder = transportClient.prepareIndex().setIndex(esObject.getIndexName())
            .setType(esObject.getTypeName());
    if (StringUtils.isNotBlank(dataId)) {
        indexRequestBuilder.setId(dataId);
    }
    // TODO 替换
    // indexRequestBuilder.setSource(esObject.getDataMap());
    indexRequestBuilder.setSource(dataBytes, XContentType.JSON);
    return indexRequestBuilder;
}
 
Example #17
Source Project: jakduk-api   Author: JakduK   File: SearchService.java    License: MIT License 6 votes vote down vote up
public void indexDocumentBoardComment(EsComment esComment) {

		String id = esComment.getId();
		String parentBoardId = esComment.getArticle().getId();

		try {
			IndexResponse response = client.prepareIndex()
					.setIndex(elasticsearchProperties.getIndexBoard())
					.setType(Constants.ES_TYPE_COMMENT)
					.setId(id)
					.setParent(parentBoardId)
					.setSource(ObjectMapperUtils.writeValueAsString(esComment), XContentType.JSON)
					.get();

		} catch (IOException e) {
			throw new ServiceException(ServiceError.ELASTICSEARCH_INDEX_FAILED, e.getCause());
		}
	}
 
Example #18
Source Project: rdf4j   Author: eclipse   File: ElasticsearchNamespaceStore.java    License: BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
@Override
public void init() {
	boolean indexExistsAlready = clientProvider.getClient()
			.admin()
			.indices()
			.exists(new IndicesExistsRequest(index))
			.actionGet()
			.isExists();

	if (!indexExistsAlready) {
		CreateIndexRequest request = new CreateIndexRequest(index);
		request.mapping(ELASTICSEARCH_TYPE, MAPPING, XContentType.JSON);
		clientProvider.getClient().admin().indices().create(request).actionGet();
	}

}
 
Example #19
Source Project: scava   Author: crossminer   File: Indexer.java    License: Eclipse Public License 2.0 6 votes vote down vote up
/**
 * Maps document Object to Json and creates and new index request
 * 
 * @param indexName
 *            - index name
 * @param documentType
 *            - document type
 * @param uid
 *            - unique identifier
 * @param object
 *            - object that represents the structure of a document for indexing
 * @return
 * @return IndexResponse
 * @throws IOException
 */
private static void index(String indexName, String documentType, String uid, String document) {

	try {
		
		GetRequest getRequest = new GetRequest(indexName, documentType, uid).fetchSourceContext(fetchSourceContext).storedFields("_none_");
		
		if(highLevelClient.exists(getRequest, getReadHeaders()))
		{
			UpdateRequest request = new UpdateRequest(indexName, documentType, uid);
			request.doc(document, XContentType.JSON);
			logger.info("Document (uid: " + uid + ") has been updated");
		}
		else
		{
			IndexRequest indexRequest = new IndexRequest();
			indexRequest .index(indexName).type(documentType).id(uid).source(document,XContentType.JSON);
				
			logger.info("Document (uid: " + uid + ") has been "	+
										highLevelClient.index(indexRequest, getWriteHeaders()).getResult().toString().toLowerCase());
		}
		
	} catch (IOException io) {
		logger.error("Method index has experienced an IO error\n" + io);
	}
}
 
Example #20
Source Project: elasticsearch-sql   Author: NLPchina   File: MainTestSuite.java    License: Apache License 2.0 5 votes vote down vote up
private static void prepareDogsIndex() {
    String dataMapping = "{  \"dog\": {" +
            " \"properties\": {\n" +
            "          \"dog_name\": {\n" +
            "            \"type\": \"text\",\n" +
            "            \"fielddata\": true\n" +
            "          }"+
            "       }"+
            "   }" +
            "}";
    client.admin().indices().preparePutMapping(TEST_INDEX_DOG).setType("dog").setSource(dataMapping, XContentType.JSON).execute().actionGet();
}
 
Example #21
Source Project: crate   Author: crate   File: CreateIndexRequest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Adds mapping that will be added when the index gets created.
 *
 * @param type   The mapping type
 * @param source The mapping source
 * @param xContentType the content type of the mapping source
 */
private CreateIndexRequest mapping(String type, BytesReference source, XContentType xContentType) {
    if (mappings.containsKey(type)) {
        throw new IllegalStateException("mappings for type \"" + type + "\" were already defined");
    }
    Objects.requireNonNull(xContentType);
    try {
        mappings.put(type, XContentHelper.convertToJson(source, xContentType));
        return this;
    } catch (IOException e) {
        throw new UncheckedIOException("failed to convert to json", e);
    }
}
 
Example #22
Source Project: elasticsearch-sql   Author: NLPchina   File: MainTestSuite.java    License: Apache License 2.0 5 votes vote down vote up
private static void prepareGameOfThronesIndex() {
    String dataMapping = "{  \"gotCharacters\": { " +
            " \"properties\": {\n" +
            " \"nickname\": {\n" +
            "\"type\":\"text\", "+
            "\"fielddata\":true"+
            "},\n"+
            " \"name\": {\n" +
            "\"properties\": {\n" +
            "\"firstname\": {\n" +
            "\"type\": \"text\",\n" +
            "  \"fielddata\": true\n" +
            "},\n" +
            "\"lastname\": {\n" +
            "\"type\": \"text\",\n" +
            "  \"fielddata\": true\n" +
            "},\n" +
            "\"ofHerName\": {\n" +
            "\"type\": \"integer\"\n" +
            "},\n" +
            "\"ofHisName\": {\n" +
            "\"type\": \"integer\"\n" +
            "}\n" +
            "}\n" +
            "}"+
            "} } }";
    client.admin().indices().preparePutMapping(TEST_INDEX_GAME_OF_THRONES).setType("gotCharacters").setSource(dataMapping, XContentType.JSON).execute().actionGet();
}
 
Example #23
Source Project: elasticsearch-sql   Author: NLPchina   File: MainTestSuite.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Loads all data from the json into the test
 * elasticsearch cluster, using TEST_INDEX
    * @param jsonPath the json file represents the bulk
    * @param defaultIndex
    * @throws Exception
 */
public static void loadBulk(String jsonPath, String defaultIndex) throws Exception {
	System.out.println(String.format("Loading file %s into elasticsearch cluster", jsonPath));

	BulkRequestBuilder bulkBuilder = client.prepareBulk();
	byte[] buffer = ByteStreams.toByteArray(new FileInputStream(jsonPath));
	bulkBuilder.add(buffer, 0, buffer.length, defaultIndex, XContentType.JSON);
	BulkResponse response = bulkBuilder.get();

	if(response.hasFailures()) {
		throw new Exception(String.format("Failed during bulk load of file %s. failure message: %s", jsonPath, response.buildFailureMessage()));
	}
}
 
Example #24
private void withContent(RestRequest restRequest, CheckedConsumer<XContentParser, IOException> withParser)
        throws IOException {
    BytesReference content = restRequest.content();
    XContentType xContentType = XContentType.JSON;
    if (content.length() > 0) {
        try (XContentParser parser = xContentType.xContent().createParser(restRequest.getXContentRegistry(),
                DeprecationHandler.THROW_UNSUPPORTED_OPERATION, content.streamInput())) {
            withParser.accept(parser);
        }
    } else {
        withParser.accept(null);
    }
}
 
Example #25
Source Project: openshift-elasticsearch-plugin   Author: fabric8io   File: SearchGuardRolesMappingACLTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSeserialization() throws Exception {
    List<RolesMapping> mappings = new RolesMappingBuilder()
        .addUser("foo", "user_of_foo")
        .expire("12345")
        .build();
    SearchGuardRolesMapping sgMapping = new SearchGuardRolesMapping();
    sgMapping.addAll(mappings);
    final String out = XContentHelper.toString(sgMapping);
    Map<String, Object> in = XContentHelper.convertToMap(new BytesArray(out), true, XContentType.JSON).v2();
    SearchGuardRolesMapping inMapping = new SearchGuardRolesMapping().load(in);
    assertEquals("Exp serialization to equal derialization", out, XContentHelper.toString(inMapping));
}
 
Example #26
Source Project: conductor   Author: Netflix   File: ElasticSearchDAOV5.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void indexWorkflow(Workflow workflow) {
    try {
        long startTime = Instant.now().toEpochMilli();
        String id = workflow.getWorkflowId();
        WorkflowSummary summary = new WorkflowSummary(workflow);
        byte[] doc = objectMapper.writeValueAsBytes(summary);

        UpdateRequest request = new UpdateRequest(indexName, WORKFLOW_DOC_TYPE, id);
        request.doc(doc, XContentType.JSON);
        request.upsert(doc, XContentType.JSON);
        request.retryOnConflict(5);

        new RetryUtil<UpdateResponse>().retryOnException(
            () -> elasticSearchClient.update(request).actionGet(),
            null,
            null,
            RETRY_COUNT,
            "Indexing workflow document: " + workflow.getWorkflowId(),
            "indexWorkflow"
        );

        long endTime = Instant.now().toEpochMilli();
        logger.debug("Time taken {} for indexing workflow: {}", endTime - startTime, workflow.getWorkflowId());
        Monitors.recordESIndexTime("index_workflow", WORKFLOW_DOC_TYPE, endTime - startTime);
        Monitors.recordWorkerQueueSize("indexQueue", ((ThreadPoolExecutor) executorService).getQueue().size());
    } catch (Exception e) {
        Monitors.error(className, "indexWorkflow");
        logger.error("Failed to index workflow: {}", workflow.getWorkflowId(), e);
    }
}
 
Example #27
Source Project: deprecated-security-advanced-modules   Author: opendistro-for-elasticsearch   File: DlsFlsFilterLeafReader.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void binaryField(final FieldInfo fieldInfo, final byte[] value) throws IOException {

    if (fieldInfo.name.equals("_source")) {
        final BytesReference bytesRef = new BytesArray(value);
        final Tuple<XContentType, Map<String, Object>> bytesRefTuple = XContentHelper.convertToMap(bytesRef, false, XContentType.JSON);
        Map<String, Object> filteredSource = bytesRefTuple.v2();
        MapUtils.deepTraverseMap(filteredSource, HASH_CB);
        final XContentBuilder xBuilder = XContentBuilder.builder(bytesRefTuple.v1().xContent()).map(filteredSource);
        delegate.binaryField(fieldInfo, BytesReference.toBytes(BytesReference.bytes(xBuilder)));
    } else {
        delegate.binaryField(fieldInfo, value);
    }
}
 
Example #28
Source Project: elasticsearch-extended-analyze   Author: johtani   File: RestExtendedAnalyzeAction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void handleRequest(final RestRequest request, final RestChannel channel, final Client client) {
    String[] text = request.paramAsStringArrayOrEmptyIfAll("text");

    ExtendedAnalyzeRequest analyzeRequest = new ExtendedAnalyzeRequest(request.param("index"));
    analyzeRequest.text(text);
    analyzeRequest.analyzer(request.param("analyzer"));
    analyzeRequest.field(request.param("field"));
    analyzeRequest.tokenizer(request.param("tokenizer"));
    analyzeRequest.tokenFilters(request.paramAsStringArray("token_filters", request.paramAsStringArray("filters", analyzeRequest.tokenFilters())));
    analyzeRequest.charFilters(request.paramAsStringArray("char_filters", analyzeRequest.charFilters()));
    analyzeRequest.attributes(request.paramAsStringArray("attributes", null));
    analyzeRequest.shortAttributeName(request.paramAsBoolean("use_short_attr", false));

    if (request.hasContent() || request.hasParam("source")) {
        XContentType type = guessBodyContentType(request);
        if (type == null) {
            if (text == null || text.length == 0) {
                text = new String[]{ RestActions.getRestContent(request).toUtf8()};
                analyzeRequest.text(text);
            }
        } else {
            buildFromContent(RestActions.getRestContent(request), analyzeRequest);
        }
    }

    client.admin().indices().execute(ExtendedAnalyzeAction.INSTANCE, analyzeRequest, new RestToXContentListener<ExtendedAnalyzeResponse>(channel));
}
 
Example #29
Source Project: vertx-elasticsearch-service   Author: hubrick   File: DefaultElasticSearchService.java    License: Apache License 2.0 5 votes vote down vote up
private void populateUpdateRequestBuilder(UpdateRequestBuilder builder, UpdateOptions options) {
    if (options != null) {
        if (options.getRouting() != null) builder.setRouting(options.getRouting());
        if (options.getParent() != null) builder.setParent(options.getParent());
        if (options.getRefreshPolicy() != null)
            builder.setRefreshPolicy(WriteRequest.RefreshPolicy.valueOf(options.getRefreshPolicy().name()));
        if (options.getWaitForActiveShard() != null)
            builder.setWaitForActiveShards(options.getWaitForActiveShard());
        if (options.getVersion() != null) builder.setVersion(options.getVersion());
        if (options.getVersionType() != null) builder.setVersionType(options.getVersionType());
        if (options.getTimeout() != null) builder.setTimeout(options.getTimeout());

        if (options.getRetryOnConflict() != null) builder.setRetryOnConflict(options.getRetryOnConflict());
        if (options.getDoc() != null) builder.setDoc(options.getDoc().encode(), XContentType.JSON);
        if (options.getUpsert() != null) builder.setUpsert(options.getUpsert().encode(), XContentType.JSON);
        if (options.getDocAsUpsert() != null) builder.setDocAsUpsert(options.getDocAsUpsert());
        if (options.getDetectNoop() != null) builder.setDetectNoop(options.getDetectNoop());
        if (options.getScriptedUpsert() != null) builder.setScriptedUpsert(options.getScriptedUpsert());

        if (options.getScript() != null) {
            builder.setScript(createScript(Optional.ofNullable(options.getScriptType()), Optional.ofNullable(options.getScriptLang()), Optional.ofNullable(options.getScriptParams()), options.getScript()));

        }
        if (!options.getFields().isEmpty()) {
            builder.setFields(options.getFields().toArray(new String[options.getFields().size()]));
        }
    }
}
 
Example #30
Source Project: deprecated-security-advanced-modules   Author: opendistro-for-elasticsearch   File: FlsTest.java    License: Apache License 2.0 5 votes vote down vote up
protected void populateData(TransportClient tc) {

        tc.index(new IndexRequest("deals").type("deals").id("0").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                .source("{\"customer\": {\"name\":\"cust1\"}, \"zip\": \"12345\",\"secret\": \"tellnoone\",\"amount\": 10}", XContentType.JSON)).actionGet();
        tc.index(new IndexRequest("deals").type("deals").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE)
                .source("{\"customer\": {\"name\":\"cust2\", \"ctype\":\"industry\"}, \"amount\": 1500}", XContentType.JSON)).actionGet();
    }