Java Code Examples for org.elasticsearch.common.xcontent.XContentType#JSON

The following examples show how to use org.elasticsearch.common.xcontent.XContentType#JSON . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testParseDynamicNullArray() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    // parse source with null array
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("new_array_field").nullValue().endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
    assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
 
Example 2
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidArrayNonConvertableType() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .startObject("array_field")
        .field("type", ArrayMapper.CONTENT_TYPE)
        .startObject(ArrayMapper.INNER_TYPE)
        .field("type", "double")
        .endObject()
        .endObject()
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    expectedException.expect(MapperParsingException.class);
    expectedException.expectMessage("failed to parse field [array_field] of type [double]");

    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .array("array_field", true, false, true)
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    mapper.parse(sourceToParse);
}
 
Example 3
Source File: RequestResolver.java    From deprecated-security-advanced-modules with Apache License 2.0 5 votes vote down vote up
private static Tuple<XContentType, BytesReference> convertSource(XContentType type, BytesReference bytes) {
    if(type == null) {
        type = XContentType.JSON;
    }

    return new Tuple<XContentType, BytesReference>(type, bytes);
}
 
Example 4
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testParseNull() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "double")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter: on
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .nullField("array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));
}
 
Example 5
Source File: DocumentMapper.java    From crate with Apache License 2.0 5 votes vote down vote up
public ParsedDocument createNoopTombstoneDoc(String index, String reason) throws MapperParsingException {
    final String id = ""; // _id won't be used.
    final SourceToParse sourceToParse = new SourceToParse(index, id, new BytesArray("{}"), XContentType.JSON);
    final ParsedDocument parsedDoc = documentParser.parseDocument(sourceToParse, noopTombstoneMetadataFieldMappers).toTombstone();
    // Store the reason of a noop as a raw string in the _source field
    final BytesRef byteRef = new BytesRef(reason);
    parsedDoc.rootDoc().add(new StoredField(SourceFieldMapper.NAME, byteRef.bytes, byteRef.offset, byteRef.length));
    return parsedDoc;
}
 
Example 6
Source File: RestChannel.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
public XContentBuilder newBuilder(@Nullable BytesReference autoDetectSource, boolean useFiltering) throws IOException {
    XContentType contentType = XContentType.fromRestContentType(request.param("format", request.header("Content-Type")));
    if (contentType == null) {
        // try and guess it from the auto detect source
        if (autoDetectSource != null) {
            contentType = XContentFactory.xContentType(autoDetectSource);
        }
    }
    if (contentType == null) {
        // default to JSON
        contentType = XContentType.JSON;
    }

    String[] filters = useFiltering ? request.paramAsStringArrayOrEmptyIfAll("filter_path") :  null;
    XContentBuilder builder = new XContentBuilder(XContentFactory.xContent(contentType), bytesOutput(), filters);
    if (request.paramAsBoolean("pretty", false)) {
        builder.prettyPrint().lfAtEnd();
    }

    builder.humanReadable(request.paramAsBoolean("human", builder.humanReadable()));

    String casing = request.param("case");
    if (casing != null) {
        String msg = "Parameter 'case' has been deprecated, all responses will use underscore casing in the future";
        DEPRECATION_LOGGER.deprecated(msg);
    }
    if (casing != null && "camelCase".equals(casing)) {
        builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.CAMELCASE);
    } else {
        // we expect all REST interfaces to write results in underscore casing, so
        // no need for double casing
        builder.fieldCaseConversion(XContentBuilder.FieldCaseConversion.NONE);
    }
    return builder;
}
 
Example 7
Source File: MappingMetaData.java    From crate with Apache License 2.0 5 votes vote down vote up
public MappingMetaData(String type, Map<String, Object> mapping) throws IOException {
    this.type = type;
    this.source = new CompressedXContent(
        (builder, params) -> builder.mapContents(mapping), XContentType.JSON, ToXContent.EMPTY_PARAMS);
    Map<String, Object> withoutType = mapping;
    if (mapping.size() == 1 && mapping.containsKey(type)) {
        withoutType = (Map<String, Object>) mapping.get(type);
    }
    initMappers(withoutType);
}
 
Example 8
Source File: ElasticsearchIndexer.java    From spring-content with Apache License 2.0 5 votes vote down vote up
void ensureAttachmentPipeline() throws IOException {
	GetPipelineRequest getRequest = new GetPipelineRequest(SPRING_CONTENT_ATTACHMENT);
	GetPipelineResponse res = client.ingest().getPipeline(getRequest, RequestOptions.DEFAULT);
	if (!res.isFound()) {
		String source = "{\"description\":\"Extract attachment information encoded in Base64 with UTF-8 charset\"," +
				"\"processors\":[{\"attachment\":{\"field\":\"data\"}}]}";
		PutPipelineRequest put = new PutPipelineRequest(SPRING_CONTENT_ATTACHMENT,
				new BytesArray(source.getBytes(StandardCharsets.UTF_8)),
				XContentType.JSON);
		WritePipelineResponse wpr = client.ingest().putPipeline(put, RequestOptions.DEFAULT);
		Assert.isTrue(wpr.isAcknowledged(), "Attachment pipeline not acknowledged by server");
	}
}
 
Example 9
Source File: MappingContentBuilderTest.java    From molgenis with GNU Lesser General Public License v3.0 4 votes vote down vote up
@BeforeEach
void setUpBeforeMethod() {
  mappingContentBuilder = new MappingContentBuilder(XContentType.JSON);
}
 
Example 10
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testObjectArrayMappingNewColumn() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .field("dynamic", true)
                            .startObject("properties")
                                .startObject("s")
                                    .field("type", "keyword")
                                .endObject()
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    // child object mapper
    assertThat(mapper.objectMappers().get("array_field"), is(instanceOf(ObjectArrayMapper.class)));
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("array_field")
        .startObject()
        .field("s", "a")
        .field("new", true)
        .endObject()
        .endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);

    Mapping mappingUpdate = doc.dynamicMappingsUpdate();
    assertThat(mappingUpdate, notNullValue());
    mapper = mapper.merge(mappingUpdate, true);
    assertThat(doc.docs().size(), is(1));
    String[] values = doc.docs().get(0).getValues("array_field.new");
    assertThat(values, arrayContainingInAnyOrder(is("T"), is("1")));
    String mappingSourceString = new CompressedXContent(mapper, XContentType.JSON, ToXContent.EMPTY_PARAMS).string();
    assertThat(
        mappingSourceString,
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"dynamic\":\"true\"," +
           "\"properties\":{" +
           "\"new\":{\"type\":\"boolean\"}," +
           "\"s\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}" +
           "}}}"));
}
 
Example 11
Source File: SettingsContentBuilder.java    From molgenis with GNU Lesser General Public License v3.0 4 votes vote down vote up
SettingsContentBuilder() {
  this(XContentType.JSON);
}
 
Example 12
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testSimpleArrayMapping() throws Exception {
    // @formatter:off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter:on
    DocumentMapper mapper = mapper(INDEX, mapping);

    assertThat(mapper.mappers().getMapper("array_field"), is(instanceOf(ArrayMapper.class)));

    BytesReference bytesReference = BytesReference.bytes(JsonXContent.contentBuilder()
        .startObject()
        .array("array_field", "a", "b", "c")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.dynamicMappingsUpdate() == null, is(true));
    assertThat(doc.docs().size(), is(1));

    ParseContext.Document fields = doc.docs().get(0);
    Set<String> values = uniqueValuesFromFields(fields, "array_field");
    assertThat(values, Matchers.containsInAnyOrder("a", "b", "c"));
    assertThat(
        mapper.mappingSource().string(),
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}}"));
}
 
Example 13
Source File: RandomObjects.java    From crate with Apache License 2.0 4 votes vote down vote up
/**
 * Returns a tuple containing random stored field values and their corresponding expected values once printed out
 * via {@link org.elasticsearch.common.xcontent.ToXContent#toXContent(XContentBuilder, ToXContent.Params)} and parsed back via
 * {@link org.elasticsearch.common.xcontent.XContentParser#objectText()}.
 * Generates values based on what can get printed out. Stored fields values are retrieved from lucene and converted via
 * {@link org.elasticsearch.index.mapper.MappedFieldType#valueForDisplay(Object)} to either strings, numbers or booleans.
 *
 * @param random Random generator
 * @param xContentType the content type, used to determine what the expected values are for float numbers.
 */
public static Tuple<List<Object>, List<Object>> randomStoredFieldValues(Random random, XContentType xContentType) {
    int numValues = randomIntBetween(random, 1, 5);
    List<Object> originalValues = new ArrayList<>();
    List<Object> expectedParsedValues = new ArrayList<>();
    int dataType = randomIntBetween(random, 0, 8);
    for (int i = 0; i < numValues; i++) {
        switch(dataType) {
            case 0:
                long randomLong = random.nextLong();
                originalValues.add(randomLong);
                expectedParsedValues.add(randomLong);
                break;
            case 1:
                int randomInt = random.nextInt();
                originalValues.add(randomInt);
                expectedParsedValues.add(randomInt);
                break;
            case 2:
                Short randomShort = (short) random.nextInt();
                originalValues.add(randomShort);
                expectedParsedValues.add(randomShort.intValue());
                break;
            case 3:
                Byte randomByte = (byte)random.nextInt();
                originalValues.add(randomByte);
                expectedParsedValues.add(randomByte.intValue());
                break;
            case 4:
                double randomDouble = random.nextDouble();
                originalValues.add(randomDouble);
                expectedParsedValues.add(randomDouble);
                break;
            case 5:
                Float randomFloat = random.nextFloat();
                originalValues.add(randomFloat);
                if (xContentType == XContentType.SMILE) {
                    //with SMILE we get back a double (this will change in Jackson 2.9 where it will return a Float)
                    expectedParsedValues.add(randomFloat.doubleValue());
                } else {
                    //with JSON AND YAML we get back a double, but with float precision.
                    expectedParsedValues.add(Double.parseDouble(randomFloat.toString()));
                }
                break;
            case 6:
                boolean randomBoolean = random.nextBoolean();
                originalValues.add(randomBoolean);
                expectedParsedValues.add(randomBoolean);
                break;
            case 7:
                String randomString = random.nextBoolean() ? RandomStrings.randomAsciiLettersOfLengthBetween(random, 3, 10) :
                        randomUnicodeOfLengthBetween(random, 3, 10);
                originalValues.add(randomString);
                expectedParsedValues.add(randomString);
                break;
            case 8:
                byte[] randomBytes = RandomStrings.randomUnicodeOfLengthBetween(random, 10, 50).getBytes(StandardCharsets.UTF_8);
                BytesArray randomBytesArray = new BytesArray(randomBytes);
                originalValues.add(randomBytesArray);
                if (xContentType == XContentType.JSON || xContentType == XContentType.YAML) {
                    //JSON and YAML write the base64 format
                    expectedParsedValues.add(Base64.getEncoder().encodeToString(randomBytes));
                } else {
                    //SMILE and CBOR write the original bytes as they support binary format
                    expectedParsedValues.add(randomBytesArray);
                }
                break;
            default:
                throw new UnsupportedOperationException();
        }
    }
    return Tuple.tuple(originalValues, expectedParsedValues);
}
 
Example 14
Source File: Elasticsearch6DynamicSink.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public SinkFunctionProvider getSinkRuntimeProvider(Context context) {
	return () -> {
		SerializationSchema<RowData> format = this.format.createRuntimeEncoder(context, schema.toRowDataType());

		final RowElasticsearchSinkFunction upsertFunction =
			new RowElasticsearchSinkFunction(
				IndexGeneratorFactory.createIndexGenerator(config.getIndex(), schema),
				config.getDocumentType(),
				format,
				XContentType.JSON,
				REQUEST_FACTORY,
				KeyExtractor.createKeyExtractor(schema, config.getKeyDelimiter())
			);

		final ElasticsearchSink.Builder<RowData> builder = builderProvider.createBuilder(
			config.getHosts(),
			upsertFunction);

		builder.setFailureHandler(config.getFailureHandler());
		builder.setBulkFlushMaxActions(config.getBulkFlushMaxActions());
		builder.setBulkFlushMaxSizeMb((int) (config.getBulkFlushMaxByteSize() >> 20));
		builder.setBulkFlushInterval(config.getBulkFlushInterval());
		builder.setBulkFlushBackoff(config.isBulkFlushBackoffEnabled());
		config.getBulkFlushBackoffType().ifPresent(builder::setBulkFlushBackoffType);
		config.getBulkFlushBackoffRetries().ifPresent(builder::setBulkFlushBackoffRetries);
		config.getBulkFlushBackoffDelay().ifPresent(builder::setBulkFlushBackoffDelay);

		// we must overwrite the default factory which is defined with a lambda because of a bug
		// in shading lambda serialization shading see FLINK-18006
		builder.setRestClientFactory(new DefaultRestClientFactory(config.getPathPrefix().orElse(null)));

		final ElasticsearchSink<RowData> sink = builder.build();

		if (config.isDisableFlushOnCheckpoint()) {
			sink.disableFlushOnCheckpoint();
		}

		return sink;
	};
}
 
Example 15
Source File: JsonXContentGenerator.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public XContentType contentType() {
    return XContentType.JSON;
}
 
Example 16
Source File: TransportShardUpsertAction.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
protected WriteReplicaResult<ShardUpsertRequest> processRequestItemsOnReplica(IndexShard indexShard, ShardUpsertRequest request) throws IOException {
    Translog.Location location = null;
    for (ShardUpsertRequest.Item item : request.items()) {
        if (item.source() == null) {
            if (logger.isTraceEnabled()) {
                logger.trace("[{} (R)] Document with id {}, has no source, primary operation must have failed",
                    indexShard.shardId(), item.id());
            }
            continue;
        }
        SourceToParse sourceToParse = new SourceToParse(
            indexShard.shardId().getIndexName(),
            item.id(),
            item.source(),
            XContentType.JSON
        );

        Engine.IndexResult indexResult = indexShard.applyIndexOperationOnReplica(
            item.seqNo(),
            item.version(),
            Translog.UNSET_AUTO_GENERATED_TIMESTAMP,
            false,
            sourceToParse
        );
        if (indexResult.getResultType() == Engine.Result.Type.MAPPING_UPDATE_REQUIRED) {
            // Even though the primary waits on all nodes to ack the mapping changes to the master
            // (see MappingUpdatedAction.updateMappingOnMaster) we still need to protect against missing mappings
            // and wait for them. The reason is concurrent requests. Request r1 which has new field f triggers a
            // mapping update. Assume that that update is first applied on the primary, and only later on the replica
            // (it’s happening concurrently). Request r2, which now arrives on the primary and which also has the new
            // field f might see the updated mapping (on the primary), and will therefore proceed to be replicated
            // to the replica. When it arrives on the replica, there’s no guarantee that the replica has already
            // applied the new mapping, so there is no other option than to wait.
            throw new TransportReplicationAction.RetryOnReplicaException(indexShard.shardId(),
                "Mappings are not available on the replica yet, triggered update: " + indexResult.getRequiredMappingUpdate());
        }
        location = indexResult.getTranslogLocation();
    }
    return new WriteReplicaResult<>(request, location, null, indexShard, logger);
}
 
Example 17
Source File: JsonXContentParser.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public XContentType contentType() {
    return XContentType.JSON;
}
 
Example 18
Source File: JsonSettingsLoader.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
@Override
public XContentType contentType() {
    return XContentType.JSON;
}
 
Example 19
Source File: JsonXContentParser.java    From crate with Apache License 2.0 4 votes vote down vote up
@Override
public XContentType contentType() {
    return XContentType.JSON;
}
 
Example 20
Source File: Elasticsearch6UpsertTableSinkFactoryTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testBuilder() {
	final TableSchema schema = createTestSchema();

	final TestElasticsearch6UpsertTableSink testSink = new TestElasticsearch6UpsertTableSink(
		false,
		schema,
		Collections.singletonList(new Host(HOSTNAME, PORT, SCHEMA)),
		INDEX,
		DOC_TYPE,
		KEY_DELIMITER,
		KEY_NULL_LITERAL,
		new JsonRowSerializationSchema(schema.toRowType()),
		XContentType.JSON,
		new DummyFailureHandler(),
		createTestSinkOptions());

	final DataStreamMock dataStreamMock = new DataStreamMock(
			new StreamExecutionEnvironmentMock(),
			Types.TUPLE(Types.BOOLEAN, schema.toRowType()));

	testSink.emitDataStream(dataStreamMock);

	final ElasticsearchSink.Builder<Tuple2<Boolean, Row>> expectedBuilder = new ElasticsearchSink.Builder<>(
		Collections.singletonList(new HttpHost(HOSTNAME, PORT, SCHEMA)),
		new ElasticsearchUpsertSinkFunction(
			INDEX,
			DOC_TYPE,
			KEY_DELIMITER,
			KEY_NULL_LITERAL,
			new JsonRowSerializationSchema(schema.toRowType()),
			XContentType.JSON,
			Elasticsearch6UpsertTableSink.UPDATE_REQUEST_FACTORY,
			new int[0]));
	expectedBuilder.setFailureHandler(new DummyFailureHandler());
	expectedBuilder.setBulkFlushBackoff(true);
	expectedBuilder.setBulkFlushBackoffType(ElasticsearchSinkBase.FlushBackoffType.EXPONENTIAL);
	expectedBuilder.setBulkFlushBackoffDelay(123);
	expectedBuilder.setBulkFlushBackoffRetries(3);
	expectedBuilder.setBulkFlushInterval(100);
	expectedBuilder.setBulkFlushMaxActions(1000);
	expectedBuilder.setBulkFlushMaxSizeMb(1);
	expectedBuilder.setRestClientFactory(new DefaultRestClientFactory(100, "/myapp"));

	assertEquals(expectedBuilder, testSink.builder);
}