Java Code Examples for org.elasticsearch.common.bytes.BytesReference#bytes()

The following examples show how to use org.elasticsearch.common.bytes.BytesReference#bytes() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testParseDynamicEmptyArray() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    // parse source with empty array
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .array("new_array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.docs().get(0).getField("new_array_field"), is(nullValue()));
    assertThat(mapper.mappers().getMapper("new_array_field"), is(nullValue()));
}
 
Example 2
Source File: ElasticSearchRestEventClient.java    From bboss-elasticsearch with Apache License 2.0 6 votes vote down vote up
public void createIndexRequest(StringBuilder bulkBuilder, IndexNameBuilder indexNameBuilder, Event event, ElasticSearchEventSerializer elasticSearchEventSerializer) throws ElasticSearchException {

		try {
			BytesReference content = elasticSearchEventSerializer == null ? BytesReference.bytes(serializer.getContentBuilder(event) ) :
					BytesReference.bytes(elasticSearchEventSerializer.getContentBuilder(event)) ;
			Map<String, Map<String, String>> parameters = new HashMap<String, Map<String, String>>();
			Map<String, String> indexParameters = new HashMap<String, String>();
			indexParameters.put(ElasticSearchRestClient.INDEX_PARAM, ((EventIndexNameBuilder)indexNameBuilder).getIndexName(event));
			indexParameters.put(ElasticSearchRestClient.TYPE_PARAM, event.getIndexType());
			if (event.getTTL() != null && event.getTTL() > 0) {
				indexParameters.put(ElasticSearchRestClient.TTL_PARAM, Long.toString(event.getTTL()) + "ms");
			}
			parameters.put(ElasticSearchRestClient.INDEX_OPERATION_NAME, indexParameters);

			Gson gson = new Gson();

			bulkBuilder.append(gson.toJson(parameters));
			bulkBuilder.append("\n");
//	      bulkBuilder.append(content.toBytesArray().toUtf8());
			bulkBuilder.append(content.utf8ToString());
			bulkBuilder.append("\n");
		} catch (IOException e) {
			throw new ElasticSearchException(e);
		}

	}
 
Example 3
Source File: IndexFeatureStoreTests.java    From elasticsearch-learning-to-rank with Apache License 2.0 6 votes vote down vote up
public void testBadValues() throws IOException {
    Map<String, Object> map = new HashMap<>();
    XContentBuilder builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent());
    BytesReference bytes = BytesReference.bytes(builder.map(map));
    assertThat(expectThrows(IllegalArgumentException.class,
            () -> IndexFeatureStore.parse(StoredFeature.class, StoredFeature.TYPE, bytes))
            .getMessage(), equalTo("No StorableElement found."));

    builder = XContentBuilder.builder(Requests.INDEX_CONTENT_TYPE.xContent());
    map.put("featureset", LtrTestUtils.randomFeatureSet());
    BytesReference bytes2 = BytesReference.bytes(builder.map(map));
    assertThat(expectThrows(IllegalArgumentException.class,
            () -> IndexFeatureStore.parse(StoredFeature.class, StoredFeature.TYPE, bytes2))
            .getMessage(), equalTo("Expected an element of type [" + StoredFeature.TYPE + "] but" +
            " got [" + StoredFeatureSet.TYPE + "]."));
}
 
Example 4
Source File: AbstractXContentTestCase.java    From crate with Apache License 2.0 6 votes vote down vote up
public static <T> XContentTester<T> xContentTester(
        CheckedBiFunction<XContent, BytesReference, XContentParser, IOException> createParser,
        Supplier<T> instanceSupplier,
        CheckedBiConsumer<T, XContentBuilder, IOException> toXContent,
        CheckedFunction<XContentParser, T, IOException> fromXContent) {
    return new XContentTester<T>(
            createParser,
            instanceSupplier,
            (testInstance, xContentType) -> {
                try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
                    toXContent.accept(testInstance, builder);
                    return BytesReference.bytes(builder);
                }
            },
            fromXContent);
}
 
Example 5
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Test
public void testInvalidArrayNonConvertableType() throws Exception {
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject().startObject(TYPE).startObject("properties")
        .startObject("array_field")
        .field("type", ArrayMapper.CONTENT_TYPE)
        .startObject(ArrayMapper.INNER_TYPE)
        .field("type", "double")
        .endObject()
        .endObject()
        .endObject().endObject().endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);

    expectedException.expect(MapperParsingException.class);
    expectedException.expectMessage("failed to parse field [array_field] of type [double]");

    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .array("array_field", true, false, true)
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    mapper.parse(sourceToParse);
}
 
Example 6
Source File: StandardnumberMappingTests.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 6 votes vote down vote up
public void testNonStandardnumber() throws Exception {
    String mapping = copyToStringFromClasspath("mapping.json");
    DocumentMapper docMapper = createIndex("some_index")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(mapping));
    String sampleText = "Hello world";
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(0, doc.getFields("someField").length);
    // re-parse it
    String builtMapping = docMapper.mappingSource().string();
    logger.warn("testNonStandardnumber: built mapping =" + builtMapping);
    DocumentMapper docMapper2 = createIndex("some_index2")
            .mapperService().documentMapperParser()
            .parse("someType", new CompressedXContent(builtMapping));
    json = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
            .field("someField", sampleText).endObject());
    sourceToParse = SourceToParse.source("some_index2", "someType", "1", json, XContentType.JSON);
    doc = docMapper2.parse(sourceToParse).rootDoc();
    assertEquals(0, doc.getFields("someField").length);
}
 
Example 7
Source File: RestHandlerUtilsTests.java    From anomaly-detection with Apache License 2.0 5 votes vote down vote up
public void testCreateXContentParser() throws IOException {
    RestRequest request = new FakeRestRequest();
    RestChannel channel = new FakeRestChannel(request, false, 1);
    XContentBuilder builder = builder().startObject().field("test", "value").endObject();
    BytesReference bytesReference = BytesReference.bytes(builder);
    XContentParser parser = RestHandlerUtils.createXContentParser(channel, bytesReference);
    parser.close();
}
 
Example 8
Source File: AbstractXContentTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
static BytesReference insertRandomFieldsAndShuffle(BytesReference xContent, XContentType xContentType,
        boolean supportsUnknownFields, String[] shuffleFieldsExceptions, Predicate<String> randomFieldsExcludeFilter,
        CheckedBiFunction<XContent, BytesReference, XContentParser, IOException> createParserFunction) throws IOException {
    BytesReference withRandomFields;
    if (supportsUnknownFields) {
        // add a few random fields to check that the parser is lenient on new fields
        withRandomFields = XContentTestUtils.insertRandomFields(xContentType, xContent, randomFieldsExcludeFilter, random());
    } else {
        withRandomFields = xContent;
    }
    XContentParser parserWithRandonFields = createParserFunction.apply(XContentFactory.xContent(xContentType), withRandomFields);
    return BytesReference.bytes(ESTestCase.shuffleXContent(parserWithRandonFields, false, shuffleFieldsExceptions));
}
 
Example 9
Source File: RandomObjects.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a random source in a given XContentType containing a random number of fields, objects and array, with maximum depth 5.
 * The minimum number of fields per object is provided as an argument.
 *
 * @param random Random generator
 */
public static BytesReference randomSource(Random random, XContentType xContentType, int minNumFields) {
    try (XContentBuilder builder = XContentFactory.contentBuilder(xContentType)) {
        builder.startObject();
        addFields(random, builder, minNumFields, 0);
        builder.endObject();
        return BytesReference.bytes(builder);
    } catch(IOException e) {
        throw new RuntimeException(e);
    }
}
 
Example 10
Source File: ESTestCase.java    From crate with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the bytes that represent the XContent output of the provided {@link ToXContent} object, using the provided
 * {@link XContentType}. Wraps the output into a new anonymous object according to the value returned
 * by the {@link ToXContent#isFragment()} method returns. Shuffles the keys to make sure that parsing never relies on keys ordering.
 */
protected static BytesReference toShuffledXContent(ToXContent toXContent, XContentType xContentType, ToXContent.Params params,
                                                   boolean humanReadable,
                                                   CheckedBiFunction<XContent, BytesReference, XContentParser, IOException>
                                                           parserFunction,
                                                   String... exceptFieldNames) throws IOException{
    BytesReference bytes = XContentHelper.toXContent(toXContent, xContentType, params, humanReadable);
    try (XContentParser parser = parserFunction.apply(xContentType.xContent(), bytes)) {
        try (XContentBuilder builder = shuffleXContent(parser, rarely(), exceptFieldNames)) {
            return BytesReference.bytes(builder);
        }
    }
}
 
Example 11
Source File: FulltextAnalyzerResolver.java    From crate with Apache License 2.0 5 votes vote down vote up
public static BytesReference encodeSettings(Settings settings) {
    try {
        XContentBuilder builder = XContentFactory.jsonBuilder();
        builder.startObject();
        settings.toXContent(builder, new ToXContent.MapParams(Collections.emptyMap()));
        builder.endObject();
        return BytesReference.bytes(builder);
    } catch (IOException e) {
        // this is a memory stream so no real I/O happens and a IOException can't really happen at runtime
        throw new RuntimeException(e);
    }
}
 
Example 12
Source File: ReferenceMappingTests.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
public void testRefFromID() throws Exception {
    IndexService indexService = createIndex("docs", Settings.EMPTY,
            "docs", getMapping("ref-mapping-from-id.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("docs");
    BytesReference json = BytesReference.bytes(jsonBuilder().startObject()
            .field("title", "A title")
            .field("authorID", "1")
            .endObject());
    SourceToParse sourceToParse = SourceToParse.source("docs", "docs", "1", json, XContentType.JSON);
    ParseContext.Document doc = docMapper.parse(sourceToParse).rootDoc();
    assertEquals(1, doc.getFields("ref").length, 1);
    assertEquals("John Doe", doc.getFields("ref")[0].stringValue());
}
 
Example 13
Source File: LangdetectMappingTests.java    From elasticsearch-plugin-bundle with GNU Affero General Public License v3.0 5 votes vote down vote up
public void testSimpleMapping() throws Exception {
    IndexService indexService = createIndex("some_index", Settings.EMPTY,
            "someType", getMapping("simple-mapping.json"));
    DocumentMapper docMapper = indexService.mapperService().documentMapper("someType");
    String sampleText = copyToStringFromClasspath("english.txt");
    BytesReference json = BytesReference.bytes(XContentFactory.jsonBuilder()
            .startObject().field("someField", sampleText).endObject());
    SourceToParse sourceToParse = SourceToParse.source("some_index", "someType", "1", json, XContentType.JSON);
    ParsedDocument doc = docMapper.parse(sourceToParse);
    assertEquals(1, doc.rootDoc().getFields("someField").length);
    assertEquals("en", doc.rootDoc().getFields("someField")[0].stringValue());
}
 
Example 14
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Test
public void testParseNullOnObjectArray() throws Exception {
    // @formatter: on
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .startObject("properties")
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter: off
    DocumentMapper mapper = mapper(INDEX, mapping);
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .nullField("array_field")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument parsedDoc = mapper.parse(sourceToParse);
    assertThat(parsedDoc.docs().size(), is(1));
    assertThat(parsedDoc.docs().get(0).getField("array_field"), is(nullValue()));


}
 
Example 15
Source File: Utils.java    From deprecated-security-advanced-modules with Apache License 2.0 5 votes vote down vote up
public static BytesReference convertStructuredMapToBytes(Map<String, Object> structuredMap) {
    try {
        return BytesReference.bytes(JsonXContent.contentBuilder().map(structuredMap));
    } catch (IOException e) {
        throw new ElasticsearchParseException("Failed to convert map", e);
    }
}
 
Example 16
Source File: InsertSourceGen.java    From crate with Apache License 2.0 4 votes vote down vote up
default BytesReference generateSourceAndCheckConstraintsAsBytesReference(Object[] values) throws IOException {
    return BytesReference.bytes(XContentFactory.jsonBuilder().map(generateSourceAndCheckConstraints(values)));
}
 
Example 17
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testObjectArrayMappingNewColumn() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .field("dynamic", true)
                            .startObject("properties")
                                .startObject("s")
                                    .field("type", "keyword")
                                .endObject()
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    // child object mapper
    assertThat(mapper.objectMappers().get("array_field"), is(instanceOf(ObjectArrayMapper.class)));
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("array_field")
        .startObject()
        .field("s", "a")
        .field("new", true)
        .endObject()
        .endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);

    Mapping mappingUpdate = doc.dynamicMappingsUpdate();
    assertThat(mappingUpdate, notNullValue());
    mapper = mapper.merge(mappingUpdate, true);
    assertThat(doc.docs().size(), is(1));
    String[] values = doc.docs().get(0).getValues("array_field.new");
    assertThat(values, arrayContainingInAnyOrder(is("T"), is("1")));
    String mappingSourceString = new CompressedXContent(mapper, XContentType.JSON, ToXContent.EMPTY_PARAMS).string();
    assertThat(
        mappingSourceString,
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"dynamic\":\"true\"," +
           "\"properties\":{" +
           "\"new\":{\"type\":\"boolean\"}," +
           "\"s\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}" +
           "}}}"));
}
 
Example 18
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testObjectArrayMapping() throws Exception {
    // @formatter: off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "object")
                            .field("dynamic", true)
                            .startObject("properties")
                                .startObject("s")
                                    .field("type", "keyword")
                                .endObject()
                            .endObject()
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    DocumentMapper mapper = mapper(INDEX, mapping);
    // child object mapper
    assertThat(mapper.objectMappers().get("array_field"), is(instanceOf(ObjectArrayMapper.class)));
    BytesReference bytesReference = BytesReference.bytes(XContentFactory.jsonBuilder()
        .startObject()
        .startArray("array_field")
        .startObject()
        .field("s", "a")
        .endObject()
        .startObject()
        .field("s", "b")
        .endObject()
        .startObject()
        .field("s", "c")
        .endObject()
        .endArray()
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    // @formatter: off
    assertThat(doc.dynamicMappingsUpdate(), nullValue());
    assertThat(doc.docs().size(), is(1));
    assertThat(
        uniqueValuesFromFields(doc.docs().get(0), "array_field.s"),
        containsInAnyOrder("a", "b", "c"));
    assertThat(mapper.mappers().getMapper("array_field.s"), instanceOf(KeywordFieldMapper.class));
    assertThat(
        mapper.mappingSource().string(),
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"dynamic\":\"true\"," +
           "\"properties\":{" +
           "\"s\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}" +
           "}}}"));
}
 
Example 19
Source File: ESTestCaseTests.java    From crate with Apache License 2.0 4 votes vote down vote up
public void testShuffleXContentExcludeFields() throws IOException {
    XContentType xContentType = randomFrom(XContentType.values());
    try (XContentBuilder builder = XContentBuilder.builder(xContentType.xContent())) {
        builder.startObject();
        {
            builder.field("field1", "value1");
            builder.field("field2", "value2");
            {
                builder.startObject("object1");
                {
                    builder.field("inner1", "value1");
                    builder.field("inner2", "value2");
                    builder.field("inner3", "value3");
                }
                builder.endObject();
            }
            {
                builder.startObject("object2");
                {
                    builder.field("inner4", "value4");
                    builder.field("inner5", "value5");
                    builder.field("inner6", "value6");
                }
                builder.endObject();
            }
        }
        builder.endObject();
        BytesReference bytes = BytesReference.bytes(builder);
        final LinkedHashMap<String, Object> initialMap;
        try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
            initialMap = (LinkedHashMap<String, Object>)parser.mapOrdered();
        }

        List<String> expectedInnerKeys1 = Arrays.asList("inner1", "inner2", "inner3");
        Set<List<String>> distinctTopLevelKeys = new HashSet<>();
        Set<List<String>> distinctInnerKeys2 = new HashSet<>();
        for (int i = 0; i < 10; i++) {
            try (XContentParser parser = createParser(xContentType.xContent(), bytes)) {
                try (XContentBuilder shuffledBuilder = shuffleXContent(parser, randomBoolean(), "object1")) {
                    try (XContentParser shuffledParser = createParser(shuffledBuilder)) {
                        Map<String, Object> shuffledMap = shuffledParser.mapOrdered();
                        assertEquals("both maps should contain the same mappings", initialMap, shuffledMap);
                        List<String> shuffledKeys = new ArrayList<>(shuffledMap.keySet());
                        distinctTopLevelKeys.add(shuffledKeys);
                        @SuppressWarnings("unchecked")
                        Map<String, Object> innerMap1 = (Map<String, Object>)shuffledMap.get("object1");
                        List<String> actualInnerKeys1 = new ArrayList<>(innerMap1.keySet());
                        assertEquals("object1 should have been left untouched", expectedInnerKeys1, actualInnerKeys1);
                        @SuppressWarnings("unchecked")
                        Map<String, Object> innerMap2 = (Map<String, Object>)shuffledMap.get("object2");
                        List<String> actualInnerKeys2 = new ArrayList<>(innerMap2.keySet());
                        distinctInnerKeys2.add(actualInnerKeys2);
                    }
                }
            }
        }

        //out of 10 shuffling runs we expect to have at least more than 1 distinct output for both top level keys and inner object2
        assertThat(distinctTopLevelKeys.size(), greaterThan(1));
        assertThat(distinctInnerKeys2.size(), greaterThan(1));
    }
}
 
Example 20
Source File: ArrayMapperTest.java    From crate with Apache License 2.0 4 votes vote down vote up
@Test
public void testSimpleArrayMapping() throws Exception {
    // @formatter:off
    String mapping = Strings.toString(XContentFactory.jsonBuilder()
        .startObject()
            .startObject(TYPE)
                .startObject("properties")
                    .startObject("array_field")
                        .field("type", ArrayMapper.CONTENT_TYPE)
                        .startObject(ArrayMapper.INNER_TYPE)
                            .field("type", "keyword")
                        .endObject()
                    .endObject()
                .endObject()
            .endObject()
        .endObject());
    // @formatter:on
    DocumentMapper mapper = mapper(INDEX, mapping);

    assertThat(mapper.mappers().getMapper("array_field"), is(instanceOf(ArrayMapper.class)));

    BytesReference bytesReference = BytesReference.bytes(JsonXContent.contentBuilder()
        .startObject()
        .array("array_field", "a", "b", "c")
        .endObject());
    SourceToParse sourceToParse = new SourceToParse(INDEX, "abc", bytesReference, XContentType.JSON);
    ParsedDocument doc = mapper.parse(sourceToParse);
    assertThat(doc.dynamicMappingsUpdate() == null, is(true));
    assertThat(doc.docs().size(), is(1));

    ParseContext.Document fields = doc.docs().get(0);
    Set<String> values = uniqueValuesFromFields(fields, "array_field");
    assertThat(values, Matchers.containsInAnyOrder("a", "b", "c"));
    assertThat(
        mapper.mappingSource().string(),
        is("{\"default\":{" +
           "\"properties\":{" +
           "\"array_field\":{" +
           "\"type\":\"array\"," +
           "\"inner\":{" +
           "\"type\":\"keyword\"" +
           "}" +
           "}" +
           "}" +
           "}}"));
}