Java Code Examples for org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL

The following examples show how to use org.elasticsearch.spark.sql.api.java.JavaEsSparkSQL. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testEsDataFrameReadMetadata() throws Exception {
	DataFrame artists = artistsAsDataFrame();
	String target = resource("sparksql-test-scala-dataframe-read-metadata", "data", version);
	JavaEsSparkSQL.saveToEs(artists, target);

	DataFrame dataframe = sqc.read().format("es").option("es.read.metadata", "true").load(target).where("id = 1");

	// Since _metadata field isn't a part of _source,
	// we want to check that it could be fetched in any position.
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(0));
	assertEquals("MALICE MIZER", dataframe.selectExpr("_metadata['_index']", "name").takeAsList(1).get(0).get(1));
	assertEquals("MALICE MIZER", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(0));
	assertEquals("sparksql-test-scala-dataframe-read-metadata", dataframe.selectExpr("name", "_metadata['_index']").takeAsList(1).get(0).get(1));
}
 
Example 2
Source Project: rdf2x   Source File: ElasticSearchPersistor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Write a {@link DataFrame} to the specified output
 *
 * @param name name of output table
 * @param df   dataframe containing the data
 */
@Override
public void writeDataFrame(String name, DataFrame df) {
    Map<String, String> props = config.getProperties(name);
    log.info("Writing to ElasticSearch: {}", props);
    JavaEsSparkSQL.saveToEs(df, props);
}
 
Example 3
@Test
   public void testEsDataset1WriteWithId() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

       JavaEsSparkSQL.saveToEs(dataset, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}
 
Example 4
@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
    Dataset<Row> dataset = artistsAsDataset();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataset, target,
            ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}
 
Example 5
@Test
public void testEsdataFrame1Write() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
	JavaEsSparkSQL.saveToEs(dataFrame, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}
 
Example 6
@Test
public void testEsdataFrame1WriteWithId() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

	String target = resource("sparksql-test-scala-basic-write-id-mapping", "data", version);
	String docEndpoint = docEndpoint("sparksql-test-scala-basic-write-id-mapping", "data", version);

	JavaEsSparkSQL.saveToEs(dataFrame, target,
			ImmutableMap.of(ES_MAPPING_ID, "id"));
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
	assertThat(RestUtils.exists(docEndpoint + "/1"), is(true));
}
 
Example 7
@Test
public void testEsSchemaRDD1WriteWithMappingExclude() throws Exception {
	DataFrame dataFrame = artistsAsDataFrame();

    String target = resource("sparksql-test-scala-basic-write-exclude-mapping", "data", version);
    JavaEsSparkSQL.saveToEs(dataFrame, target,ImmutableMap.of(ES_MAPPING_EXCLUDE, "url"));
    assertTrue(RestUtils.exists(target));
    assertThat(RestUtils.get(target + "/_search?"), not(containsString("url")));
}
 
Example 8
@Test
   public void testEsDataset1Write() throws Exception {
       Dataset<Row> dataset = artistsAsDataset();

	String target = resource("sparksql-test-scala-basic-write", "data", version);
       JavaEsSparkSQL.saveToEs(dataset, target);
	assertTrue(RestUtils.exists(target));
	assertThat(RestUtils.get(target + "/_search?"), containsString("345"));
}
 
Example 9
Source Project: elasticsearch-rest-command   Source File: TaskRestHandler.java    License: The Unlicense 3 votes vote down vote up
private JavaSchemaRDD  getSchemaRDD(JavaSQLContext sqlContext, String command) throws ParseException, CommandException, IOException{
	CommandParser parser = new CommandParser(command);

	Search search = new Search(parser, client, logger);
	
	if(search.indices.length == 0 || search.indices.length > 1)
		throw new InvalidParameterException(String.format("indices.length = %d", search.indices.length));
	
	if(search.sourceTypes.length == 0 || search.sourceTypes.length > 1)
		throw new InvalidParameterException(String.format("sourceTypes.length = %d", search.sourceTypes.length));
	
	String query = search.querySearch.toString();
	
	return JavaEsSparkSQL.esRDD(sqlContext, String.format("%s/%s", search.indices[0], search.sourceTypes[0]), query);
}