Java Code Examples for org.elasticsearch.common.xcontent.support.XContentMapValues#nodeStringValue()

The following examples show how to use org.elasticsearch.common.xcontent.support.XContentMapValues#nodeStringValue() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: GitHubRiver.java From elasticsearch-river-github with Apache License 2.0

5 votes

@SuppressWarnings({"unchecked"})
@Inject
public GitHubRiver(RiverName riverName, RiverSettings settings, Client client) {
    super(riverName, settings);
    this.client = client;

    if (!settings.settings().containsKey("github")) {
        throw new IllegalArgumentException("Need river settings - owner and repository.");
    }

    // get settings
    Map<String, Object> githubSettings = (Map<String, Object>) settings.settings().get("github");
    owner = XContentMapValues.nodeStringValue(githubSettings.get("owner"), null);
    repository = XContentMapValues.nodeStringValue(githubSettings.get("repository"), null);

    index = String.format("%s&%s", owner, repository);
    userRequestedInterval = XContentMapValues.nodeIntegerValue(githubSettings.get("interval"), 60);

    // auth (optional)
    username = null;
    password = null;
    if (githubSettings.containsKey("authentication")) {
        Map<String, Object> auth = (Map<String, Object>) githubSettings.get("authentication");
        username = XContentMapValues.nodeStringValue(auth.get("username"), null);
        password = XContentMapValues.nodeStringValue(auth.get("password"), null);
    }

    // endpoint (optional - default to github.com)
    endpoint = XContentMapValues.nodeStringValue(githubSettings.get("endpoint"), "https://api.github.com");

    logger.info("Created GitHub river.");
}

Example 2

Source File: LdapLoginSource.java From elasticsearch-imap with Apache License 2.0

5 votes

public LdapLoginSource(Map<String, Object> settings, String masterUser, String masterPassword) {
fUserNames = new ArrayList<>();
fUserPasswords = new ArrayList<>();

fMasterUser = masterUser;
fMasterPassword = masterPassword;

String url = XContentMapValues.nodeStringValue(settings.get("ldap_url"), null);
String base = XContentMapValues.nodeStringValue(settings.get("ldap_base"), null);
String user = XContentMapValues.nodeStringValue(settings.get("ldap_user"), null);
String password = XContentMapValues.nodeStringValue(settings.get("ldap_password"), null);

fConnector = new SimpleLdapConnector(url, base, user, password, true);

fNameField = XContentMapValues.nodeStringValue(settings.get("ldap_name_field"), DEF_USER_NAME_FIELD);
fPasswordField = XContentMapValues.nodeStringValue(settings.get("ldap_password_field"), DEF_USER_PW_FIELD);
fLdapFilter = fNameField + "=*";

fLogger = ESLoggerFactory.getLogger(LdapLoginSource.class.getName());

fLock = new Object();
fInitialized = false;

//start refreshing thread once initialized; interval in minutes
String refreshParam = XContentMapValues.nodeStringValue(settings.get("ldap_refresh_interval"), DEF_REFRESH_INT);
fRefreshInt = Long.parseLong(refreshParam) * 60000L;
if(fRefreshInt > 0) {
    //TODO: actually stop refreshing thread somehow
    fActive = true;
    Thread t = new Thread(this);
    t.setDaemon(true);
    t.start();
}
   }

Example 3

Source File: IMAPImporter.java From elasticsearch-imap with Apache License 2.0

5 votes

@SuppressWarnings("unchecked")
private List<String> arrayNodeToList(Object arrayNode) {
    ArrayList<String> list = new ArrayList<>();
    if(XContentMapValues.isArray(arrayNode)) {
        for(Object node : (List<Object>) arrayNode) {
            String value = XContentMapValues.nodeStringValue(node, null);
            if(value != null) {
                list.add(value);
            }
        }
    }
    return list;
}

Example 4

Source File: IMAPImporter.java From elasticsearch-imap with Apache License 2.0

5 votes

private void getUserLogins(final Map<String, Object> imapSettings) {
    String userSource = XContentMapValues.nodeStringValue(imapSettings.get("user_source"), null);
    ILoginSource source = null;

    if ("ldap".equals(userSource)) {
        //master user credentials for Dovecot
        String masterUser = XContentMapValues.nodeStringValue(imapSettings.get("master_user"), null);
        String masterPassword = XContentMapValues.nodeStringValue(imapSettings.get("master_password"), null);
        source = new LdapLoginSource(imapSettings, masterUser, masterPassword);
    } else {
        //read logins directly
        String _user = XContentMapValues.nodeStringValue(imapSettings.get("user"), null);
        String _password = XContentMapValues.nodeStringValue(imapSettings.get("password"), null);

        if (_user != null && !_user.isEmpty()) {
            users.add(_user);
            passwords.add(_password);
        }

        List<String> _users = arrayNodeToList(imapSettings.get("users"));
        List<String> _passwords = arrayNodeToList(imapSettings.get("passwords"));

        //TODO: inject master user credentials?
        if (_users != null && !_users.isEmpty()) {
            users.addAll(_users);
            passwords.addAll(_passwords);
        }
    }

    //read from generic source
    if (source != null) {
        users.addAll(source.getUserNames());
        passwords.addAll(source.getUserPasswords());
    }
}

Example 5

Source File: HBaseRiver.java From Elasticsearch-HBase-River with Apache License 2.0

5 votes

/**
 * Fetch the value of a configuration that has a default value and is therefore optional.
 * 
 * @param config Key of the configuration to fetch
 * @param defaultValue The value to set if no value could be found
 * @return
 */
@SuppressWarnings({ "unchecked" })
private String readConfig(final String config, final String defaultValue) {
	if (this.settings.settings().containsKey(CONFIG_SPACE)) {
		Map<String, Object> mysqlSettings = (Map<String, Object>) this.settings.settings().get(CONFIG_SPACE);
		return XContentMapValues.nodeStringValue(mysqlSettings.get(config), defaultValue);
	}
	return defaultValue;
}

Example 6

Source File: Neo4jDriver.java From elasticsearch-river-neo4j with Apache License 2.0

5 votes

@Inject
public Neo4jDriver(RiverName riverName, RiverSettings settings, @RiverIndexName final String riverIndexName, final Client client) {
    super(riverName, settings);
    this.client = client;

    uri = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.uri", settings.settings()), DEFAULT_NEO_URI);
    List<Object> neo4jLabels = XContentMapValues.extractRawValues("neo4j.labels", settings.settings());
    String label;
    if(XContentMapValues.isArray(neo4jLabels)) {
        for (Object neo4jLabel : neo4jLabels) {
            label = XContentMapValues.nodeStringValue(neo4jLabel, null);
            labels.add(DynamicLabel.label(label));
        }
    }
    timestampField = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.timestampField", settings.settings()), DEFAULT_NEO_TIMESTAMP_FIELD);
    interval = XContentMapValues.nodeIntegerValue(XContentMapValues.extractValue("neo4j.interval", settings.settings()), DEFAULT_NEO_INTERVAL);
    index = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name", settings.settings()), DEFAULT_NEO_INDEX);
    type = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type", settings.settings()), DEFAULT_NEO_TYPE);
    indexFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name.label",
        settings.settings()), null);
    typeFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type.label",
        settings.settings()), null);

    logger.debug("Neo4j settings [uri={}]", new Object[]{uri});
    logger.debug("River settings [indexName={}, type={}, interval={}, timestampField={}, indexLabel={}, " +
            "typelabel={}]",
        new Object[]{index,
            type,
            interval,
            timestampField,
            indexFromLabel,
            typeFromLabel}
    );

}

Example 7

Source File: IsPrimeSearchScriptFactory.java From elasticsearch-native-script-example with Apache License 2.0

5 votes

/**
 * This method is called for every search on every shard.
 *
 * @param params list of script parameters passed with the query
 * @return new native script
 */
@Override
public ExecutableScript newScript(@Nullable Map<String, Object> params) {
    // Example of a mandatory string parameter
    // The XContentMapValues helper class can be used to simplify parameter parsing
    String fieldName = params == null ? null : XContentMapValues.nodeStringValue(params.get("field"), defaultFieldName);
    if (!Strings.hasLength(fieldName)) {
        throw new IllegalArgumentException("Missing the field parameter");
    }

    // Example of an optional integer  parameter
    int certainty = params == null ? 10 : XContentMapValues.nodeIntegerValue(params.get("certainty"), 10);
    return new IsPrimeSearchScript(fieldName, certainty);
}

Example 8

Source File: FieldAliasMapper.java From crate with Apache License 2.0

5 votes

@Override
public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext)
    throws MapperParsingException {

    FieldAliasMapper.Builder builder = new FieldAliasMapper.Builder(name);
    Object pathField = node.remove(Names.PATH);
    String path = XContentMapValues.nodeStringValue(pathField, null);
    if (path == null) {
        throw new MapperParsingException("The [path] property must be specified for field [" + name + "].");
    }
    return builder.path(path);
}

Example 9

Source File: KafkaRiverConfig.java From elasticsearch-river-kafka with Apache License 2.0

4 votes

public KafkaRiverConfig(RiverSettings settings)
{
	if (settings.settings().containsKey("kafka")) {
        Map<String, Object> kafkaSettings = (Map<String, Object>) settings.settings().get("kafka");
        
        topic = (String)kafkaSettings.get("topic");
        zookeeper = XContentMapValues.nodeStringValue(kafkaSettings.get("zookeeper"), "localhost");
        factoryClass = XContentMapValues.nodeStringValue(kafkaSettings.get("message_handler_factory_class"), "org.elasticsearch.river.kafka.JsonMessageHandlerFactory");
        brokerHost = XContentMapValues.nodeStringValue(kafkaSettings.get("broker_host"), "localhost");
        brokerPort = XContentMapValues.nodeIntegerValue(kafkaSettings.get("broker_port"), 9092);
        partition = XContentMapValues.nodeIntegerValue(kafkaSettings.get("partition"), 0);
    }
	else
	{
		zookeeper = "localhost";
		brokerHost = "localhost";
		brokerPort = 9092;
		topic = "default_topic";
		partition = 0;
		factoryClass = "org.elasticsearch.river.kafka.JsonMessageHandlerFactory";
	}
    
    if (settings.settings().containsKey("index")) {
        Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
        bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size_bytes"), 10*1024*1024);
        if (indexSettings.containsKey("bulk_timeout")) {
            bulkTimeout = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"), TimeValue.timeValueMillis(10000));
        } else {
            bulkTimeout = TimeValue.timeValueMillis(10);
        }
    } else {
        bulkSize = 10*1024*1024;
        bulkTimeout = TimeValue.timeValueMillis(10000);
    }
    
    if (settings.settings().containsKey("statsd")) {
        Map<String, Object> statsdSettings = (Map<String, Object>) settings.settings().get("statsd");
        statsdHost = (String)statsdSettings.get("host");
        statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get("port"), 8125);
        statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get("prefix"), "es-kafka-river");
    }
	else
	{
		statsdHost = null;
		statsdPort = -1;
		statsdPrefix = null;
	}
}

Example 10

Source File: S3River.java From es-amazon-s3-river with Apache License 2.0

4 votes

@Inject
@SuppressWarnings({ "unchecked" })
protected S3River(RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) throws Exception{
   super(riverName, settings);
   this.client = client;
   this.threadPool = threadPool;
   this.riverStatus = RiverStatus.UNKNOWN;
   
   // Deal with connector settings.
   if (settings.settings().containsKey("amazon-s3")){
      Map<String, Object> feed = (Map<String, Object>)settings.settings().get("amazon-s3");
      
      // Retrieve feed settings.
      String feedname = XContentMapValues.nodeStringValue(feed.get("name"), null);
      String bucket = XContentMapValues.nodeStringValue(feed.get("bucket"), null);
      String pathPrefix = XContentMapValues.nodeStringValue(feed.get("pathPrefix"), null);
      String downloadHost = XContentMapValues.nodeStringValue(feed.get("download_host"), null);
      int updateRate = XContentMapValues.nodeIntegerValue(feed.get("update_rate"), 15 * 60 * 1000);
      boolean jsonSupport = XContentMapValues.nodeBooleanValue(feed.get("json_support"), false);
      double indexedCharsRatio  = XContentMapValues.nodeDoubleValue(feed.get("indexed_chars_ratio"), 0.0);
      
      String[] includes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.includes");
      String[] excludes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.excludes");
      
      // Retrieve connection settings.
      String accessKey = XContentMapValues.nodeStringValue(feed.get("accessKey"), null);
      String secretKey = XContentMapValues.nodeStringValue(feed.get("secretKey"), null);
      boolean useIAMRoleForEC2 = XContentMapValues.nodeBooleanValue(feed.get("use_EC2_IAM"), false);
      
      feedDefinition = new S3RiverFeedDefinition(feedname, bucket, pathPrefix, downloadHost,
            updateRate, Arrays.asList(includes), Arrays.asList(excludes), accessKey, secretKey, useIAMRoleForEC2,
            jsonSupport, indexedCharsRatio);
   } else {
      logger.error("You didn't define the amazon-s3 settings. Exiting... See https://github.com/lbroudoux/es-amazon-s3-river");
      indexName = null;
      typeName = null;
      bulkSize = 100;
      feedDefinition = null;
      s3 = null;
      return;
   }
   
   // Deal with index settings if provided.
   if (settings.settings().containsKey("index")) {
      Map<String, Object> indexSettings = (Map<String, Object>)settings.settings().get("index");
      
      indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
      typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), S3RiverUtil.INDEX_TYPE_DOC);
      bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
   } else {
      indexName = riverName.name();
      typeName = S3RiverUtil.INDEX_TYPE_DOC;
      bulkSize = 100;
   }
   
   // We need to connect to Amazon S3 after ensure mandatory settings are here.
   if (feedDefinition.getBucket() == null){
      logger.error("Amazon S3 bucket should not be null. Please fix this.");
      throw new IllegalArgumentException("Amazon S3 bucket should not be null.");
   }
   // Connect using the appropriate authentication process.
   if (feedDefinition.getAccessKey() == null && feedDefinition.getSecretKey() == null) {
      s3 = new S3Connector(feedDefinition.isUseIAMRoleForEC2());
   } else {
      s3 = new S3Connector(feedDefinition.getAccessKey(), feedDefinition.getSecretKey());
   }
   try {
      s3.connectUserBucket(feedDefinition.getBucket(), feedDefinition.getPathPrefix());
   } catch (AmazonS3Exception ase){
      logger.error("Exception while connecting Amazon S3 user bucket. "
            + "Either access key, secret key, IAM Role or bucket name are incorrect");
      throw ase;
   }

   this.riverStatus = RiverStatus.INITIALIZED;
}

Example 11

Source File: RiverConfig.java From elasticsearch-river-kafka with Apache License 2.0

4 votes

public RiverConfig(RiverName riverName, RiverSettings riverSettings) {

        // Extract kafka related configuration
        if (riverSettings.settings().containsKey("kafka")) {
            Map<String, Object> kafkaSettings = (Map<String, Object>) riverSettings.settings().get("kafka");

            topic = (String) kafkaSettings.get(TOPIC);
            zookeeperConnect = XContentMapValues.nodeStringValue(kafkaSettings.get(ZOOKEEPER_CONNECT), "localhost");
            zookeeperConnectionTimeout = XContentMapValues.nodeIntegerValue(kafkaSettings.get(ZOOKEEPER_CONNECTION_TIMEOUT), 10000);
            messageType = MessageType.fromValue(XContentMapValues.nodeStringValue(kafkaSettings.get(MESSAGE_TYPE),
                    MessageType.JSON.toValue()));
        } else {
            zookeeperConnect = "localhost";
            zookeeperConnectionTimeout = 10000;
            topic = "elasticsearch-river-kafka";
            messageType = MessageType.JSON;
        }

        // Extract ElasticSearch related configuration
        if (riverSettings.settings().containsKey("index")) {
            Map<String, Object> indexSettings = (Map<String, Object>) riverSettings.settings().get("index");
            indexName = XContentMapValues.nodeStringValue(indexSettings.get(INDEX_NAME), riverName.name());
            typeName = XContentMapValues.nodeStringValue(indexSettings.get(MAPPING_TYPE), "status");
            bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get(BULK_SIZE), 100);
            concurrentRequests = XContentMapValues.nodeIntegerValue(indexSettings.get(CONCURRENT_REQUESTS), 1);
            actionType = ActionType.fromValue(XContentMapValues.nodeStringValue(indexSettings.get(ACTION_TYPE),
                    ActionType.INDEX.toValue()));
            flushInterval = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get(FLUSH_INTERVAL), "12h"), FLUSH_12H);
        } else {
            indexName = riverName.name();
            typeName = "status";
            bulkSize = 100;
            concurrentRequests = 1;
            actionType = ActionType.INDEX;
            flushInterval = FLUSH_12H;
        }
        
        // Extract StatsD related configuration
        if (riverSettings.settings().containsKey("statsd")) {
            Map<String, Object> statsdSettings = (Map<String, Object>) riverSettings.settings().get("statsd");
            statsdHost = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_HOST), "localhost");
            statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_PREFIX), "kafka_river");
            statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_PORT), 8125);
            statsdIntervalInSeconds = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_INTERVAL_IN_SECONDS), 10);
        }
    }

Example 12

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

3 votes

protected void renameScenarioIMAP(final String resource) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        createInitialIMAPTestdata(props, user, password, 10, true); // 3*10

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(25 * 1000);

        Assert.assertEquals(30, getCount(indexName, typeName));

        renameMailbox(props, "ES-IMAP-RIVER-TESTS", user, password);

        Thread.sleep(25 * 1000);

        deleteMailsFromUserMailbox(props, "renamed_from_ES-IMAP-RIVER-TESTS", 4, 2, user, password); // delete
        // 2

        Thread.sleep(25 * 1000);

        Assert.assertEquals(28, getCount(indexName, typeName));

    }

Example 13

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

2 votes

protected void deleteScenarioIMAP(final String resource, boolean keep_expunged) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        createInitialIMAPTestdata(props, user, password, 11, true); // 3*11
                                                                    // =
                                                                    // 33

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(25 * 1000);

        Assert.assertEquals(33, getCount(indexName, typeName));

        deleteMailsFromUserMailbox(props, "INBOX", 5, 3, user, password); // delete
        // 3

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?33:30, getCount(indexName, typeName));

        createInitialIMAPTestdata(props, user, password, 7, false); // 3*7=21

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?54:51, getCount(indexName, typeName));
        deleteMailsFromUserMailbox(props, "INBOX", 4, 2, user, password); // delete
        // 2

        Thread.sleep(25 * 1000);

        Assert.assertEquals(keep_expunged?54:49, getCount(indexName, typeName));

    }

Example 14

Source File: AbstractIMAPRiverScenario.java From elasticsearch-imap with Apache License 2.0

2 votes

protected void deleteScenarioPOP(final String resource) throws Exception {

        Map<String, Object> settings = settings("/" + resource);

        final Properties props = new Properties();
        final String indexName = XContentMapValues.nodeStringValue(settings.get("mail_index_name"), "imapriver");
        final String typeName = XContentMapValues.nodeStringValue(settings.get("mail_type_name"), "mail");
        final String user = XContentMapValues.nodeStringValue(settings.get("user"), null);
        final String password = XContentMapValues.nodeStringValue(settings.get("password"), null);

        for (final Map.Entry<String, Object> entry : settings.entrySet()) {

            if (entry != null && entry.getKey().startsWith("mail.")) {
                props.setProperty(entry.getKey(), String.valueOf(entry.getValue()));
            }
        }

        putMailInMailbox(10);

        registerRiver("myrivunit", resource);

        Thread.sleep(1000);

        Thread.sleep(5 * 1000);

        deleteMailsFromUserMailbox(props, "INBOX", 1, 3, user, password); // delete
        // 3

        Thread.sleep(5 * 1000);

        putMailInMailbox(5);

        Thread.sleep(5 * 1000);

        deleteMailsFromUserMailbox(props, "INBOX", 4, 2, user, password); // delete
        // 2

        Thread.sleep(5 * 1000);

        Assert.assertEquals(10, getCount(indexName, typeName));

    }