org.elasticsearch.river.RiverSettings Java Examples

public void testDefaults()
{
	Map<String, Object> map = new HashMap<>();		
	RiverSettings settings = new RiverSettings(
		ImmutableSettings.settingsBuilder().build(), 
		map);
	KafkaRiverConfig c = new KafkaRiverConfig(settings);
	
	assertEquals("localhost", c.brokerHost);
	assertEquals(9092, c.brokerPort);
	assertEquals("localhost", c.zookeeper);
	assertEquals(10485760, c.bulkSize);
	assertEquals(0, c.partition);
	assertEquals("default_topic", c.topic);
	assertEquals(10000, c.bulkTimeout.millis());
	
	assertEquals("org.elasticsearch.river.kafka.JsonMessageHandlerFactory", c.factoryClass);
	
	assertEquals(-1, c.statsdPort);
	assertNull(c.statsdHost);
	assertNull(c.statsdPrefix);
	
}
 

@SuppressWarnings({"unchecked"})
@Inject
public GitHubRiver(RiverName riverName, RiverSettings settings, Client client) {
    super(riverName, settings);
    this.client = client;

    if (!settings.settings().containsKey("github")) {
        throw new IllegalArgumentException("Need river settings - owner and repository.");
    }

    // get settings
    Map<String, Object> githubSettings = (Map<String, Object>) settings.settings().get("github");
    owner = XContentMapValues.nodeStringValue(githubSettings.get("owner"), null);
    repository = XContentMapValues.nodeStringValue(githubSettings.get("repository"), null);

    index = String.format("%s&%s", owner, repository);
    userRequestedInterval = XContentMapValues.nodeIntegerValue(githubSettings.get("interval"), 60);

    // auth (optional)
    username = null;
    password = null;
    if (githubSettings.containsKey("authentication")) {
        Map<String, Object> auth = (Map<String, Object>) githubSettings.get("authentication");
        username = XContentMapValues.nodeStringValue(auth.get("username"), null);
        password = XContentMapValues.nodeStringValue(auth.get("password"), null);
    }

    // endpoint (optional - default to github.com)
    endpoint = XContentMapValues.nodeStringValue(githubSettings.get("endpoint"), "https://api.github.com");

    logger.info("Created GitHub river.");
}
 

@Inject
  public KafkaRiver(RiverName riverName, RiverSettings settings, Client client) {
      super(riverName, settings);
      this.client = client;
      
      try {
	logger.info("KafkaRiver created: name={}, type={}", riverName.getName(), riverName.getType());
	this.riverConfig = new KafkaRiverConfig(settings);
} catch (Exception e) {
	logger.error("Unexpected Error occurred", e);
	throw new RuntimeException(e);
}
  }
 

public void testNormal() {
	Settings globalSettings = ImmutableSettings.settingsBuilder().put("cluster.name", "jason-hfs-cluster").build();
	Map<String, Object> config = new HashMap<>();
	config.put("statsd", ImmutableMap.<String, Object> builder().put("host", "localhost").put("port", "1234").put("prefix", "my_prefix").build());

	StatsReporter r = new StatsReporter(new KafkaRiverConfig(new RiverSettings(globalSettings, config)));
	assertTrue(r.isEnabled());

}
 

public void testNotConfigured() {
	Settings globalSettings = ImmutableSettings.settingsBuilder().put("cluster.name", "jason-hfs-cluster").build();
	Map<String, Object> config = new HashMap<>();
	// no statsd config at all
	assertFalse(new StatsReporter(new KafkaRiverConfig(new RiverSettings(globalSettings, config))).isEnabled());

	// missing host
	config.put("statsd", ImmutableMap.<String, Object> builder().put("port", "1234").put("prefix", "my_prefix").build());
	assertFalse(new StatsReporter(new KafkaRiverConfig(new RiverSettings(globalSettings, config))).isEnabled());
}
 

/**
 * Loads and verifies all the configuration needed to run this river.
 * 
 * @param riverName
 * @param settings
 * @param esClient
 */
@Inject
public HBaseRiver(final RiverName riverName, final RiverSettings settings, final Client esClient) {
	super(riverName, settings);
	this.esClient = esClient;
	this.logger.info("Creating HBase Stream River");

	this.normalizeFields = Boolean.parseBoolean(readConfig("normalizeFields", "true"));
	this.hosts = readConfig("hosts");
	this.table = readConfig("table");
	this.columnSeparator = readConfig("columnSeparator", null);
	this.idField = normalizeField(readConfig("idField", null));
	this.index = normalizeField(readConfig("index", riverName.name()));
	this.type = normalizeField(readConfig("type", this.table));
	this.interval = Long.parseLong(readConfig("interval", "600000"));
	this.batchSize = Integer.parseInt(readConfig("batchSize", "100"));
	this.charset = Charset.forName(readConfig("charset", "UTF-8"));
	this.deleteOld = Boolean.parseBoolean(readConfig("deleteOld", "false"));

	final String family = readConfig("family", null);
	this.family = family != null ? family.getBytes(this.charset) : null;
	this.qualifiers = readConfig("qualifiers", null);
	this.customMapping = readConfig("customMapping", null);

	if (this.interval <= 0) {
		throw new IllegalArgumentException("The interval between runs must be at least 1 ms. The current config is set to "
				+ this.interval);
	}
	if (this.batchSize <= 0) {
		throw new IllegalArgumentException("The batch size must be set to at least 1. The current config is set to " + this.batchSize);
	}
}
 

@Inject
public Neo4jDriver(RiverName riverName, RiverSettings settings, @RiverIndexName final String riverIndexName, final Client client) {
    super(riverName, settings);
    this.client = client;

    uri = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.uri", settings.settings()), DEFAULT_NEO_URI);
    List<Object> neo4jLabels = XContentMapValues.extractRawValues("neo4j.labels", settings.settings());
    String label;
    if(XContentMapValues.isArray(neo4jLabels)) {
        for (Object neo4jLabel : neo4jLabels) {
            label = XContentMapValues.nodeStringValue(neo4jLabel, null);
            labels.add(DynamicLabel.label(label));
        }
    }
    timestampField = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("neo4j.timestampField", settings.settings()), DEFAULT_NEO_TIMESTAMP_FIELD);
    interval = XContentMapValues.nodeIntegerValue(XContentMapValues.extractValue("neo4j.interval", settings.settings()), DEFAULT_NEO_INTERVAL);
    index = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name", settings.settings()), DEFAULT_NEO_INDEX);
    type = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type", settings.settings()), DEFAULT_NEO_TYPE);
    indexFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.name.label",
        settings.settings()), null);
    typeFromLabel = XContentMapValues.nodeStringValue(XContentMapValues.extractValue("index.type.label",
        settings.settings()), null);

    logger.debug("Neo4j settings [uri={}]", new Object[]{uri});
    logger.debug("River settings [indexName={}, type={}, interval={}, timestampField={}, indexLabel={}, " +
            "typelabel={}]",
        new Object[]{index,
            type,
            interval,
            timestampField,
            indexFromLabel,
            typeFromLabel}
    );

}
 

@Test
public void settingsAreTakenFromNeo4jObjectIfSet() throws IOException {
    InputStream in = this.classLoader.getResourceAsStream("dummy_river_settings.json");
    RiverSettings riverSettings = new RiverSettings(ImmutableSettings.settingsBuilder().build(), XContentHelper.convertToMap(
                Streams.copyToByteArray(in), false).v2());
    Neo4jDriver driver = new Neo4jDriver(name, riverSettings, "myindex", client);

    assertEquals("time", driver.getTimestampField());
    assertEquals("neoindex", driver.getIndex());
    assertEquals(500, driver.getInterval());
    assertEquals("http://192.56.57.89:7888/db/data", driver.getUri());
    assertEquals("turtle", driver.getType());
}
 

@Test
public void settingsAreDefaultsIfJsonObjectNotSet() {

    Map<String, Object> map = new HashMap<String, Object>();
    RiverSettings settings = new RiverSettings(mock(Settings.class), map);
    Neo4jDriver driver = new Neo4jDriver(name, settings, "myindex", client);

    assertEquals(Neo4jDriver.DEFAULT_NEO_URI, driver.getUri());
    assertEquals(Neo4jDriver.DEFAULT_NEO_TIMESTAMP_FIELD, driver.getTimestampField());
    assertEquals(Neo4jDriver.DEFAULT_NEO_INDEX, driver.getIndex());
    assertEquals(Neo4jDriver.DEFAULT_NEO_INTERVAL, driver.getInterval());
    assertEquals(Neo4jDriver.DEFAULT_NEO_TYPE, driver.getType());
}
 

@Test
public void closingRiverShutsDownExecutor() {

    Map<String, Object> map = new HashMap<String, Object>();
    RiverSettings settings = new RiverSettings(mock(Settings.class), map);
    Neo4jDriver driver = new Neo4jDriver(name, settings, "myindex", client);

    assertNull(driver.executor);
    driver.start();
    assertNotNull(driver.executor);
    assertFalse(driver.executor.isShutdown());
    driver.close();
    assertTrue(driver.executor.isShutdown());
}
 

@Test
public void closingNonStartedRiverShutsDownWithoutException() {

    Map<String, Object> map = new HashMap<String, Object>();
    RiverSettings settings = new RiverSettings(mock(Settings.class), map);
    Neo4jDriver driver = new Neo4jDriver(name, settings, "myindex", client);
    driver.close();
}
 

@Inject
protected KafkaRiver(final RiverName riverName, final RiverSettings riverSettings, final Client client) {
    super(riverName, riverSettings);

    riverConfig = new RiverConfig(riverName, riverSettings);
    kafkaConsumer = new KafkaConsumer(riverConfig);
    stats = new Stats();
    
    if(null != riverConfig.getStatsdHost()) {
        logger.debug("Found statsd configuration. Starting client (prefix={}, host={}, port={}, interval={})",
                riverConfig.getStatsdPrefix(), riverConfig.getStatsdHost(), riverConfig.getStatsdPort(),
                riverConfig.getStatsdIntervalInSeconds());

        statsAgent = new StatsAgent(riverConfig);

        int intervalInMs = riverConfig.getStatsdIntervalInSeconds() * 1000;
        timer = new Timer();
        timer.scheduleAtFixedRate(new LogStatsTask(), intervalInMs, intervalInMs);
    }
    else {
        logger.debug("No statsd configuration found. Will not report stats...");
    }

    switch (riverConfig.getActionType()) {
        case INDEX:
            elasticsearchProducer = new IndexDocumentProducer(client, riverConfig, kafkaConsumer, stats);
            break;
        case DELETE:
            elasticsearchProducer = new DeleteDocumentProducer(client, riverConfig, kafkaConsumer, stats);
            break;
        case RAW_EXECUTE:
            elasticsearchProducer = new RawMessageProducer(client, riverConfig, kafkaConsumer, stats);
            break;
    }
}
 

public KafkaRiverConfig(RiverSettings settings)
{
	if (settings.settings().containsKey("kafka")) {
        Map<String, Object> kafkaSettings = (Map<String, Object>) settings.settings().get("kafka");
        
        topic = (String)kafkaSettings.get("topic");
        zookeeper = XContentMapValues.nodeStringValue(kafkaSettings.get("zookeeper"), "localhost");
        factoryClass = XContentMapValues.nodeStringValue(kafkaSettings.get("message_handler_factory_class"), "org.elasticsearch.river.kafka.JsonMessageHandlerFactory");
        brokerHost = XContentMapValues.nodeStringValue(kafkaSettings.get("broker_host"), "localhost");
        brokerPort = XContentMapValues.nodeIntegerValue(kafkaSettings.get("broker_port"), 9092);
        partition = XContentMapValues.nodeIntegerValue(kafkaSettings.get("partition"), 0);
    }
	else
	{
		zookeeper = "localhost";
		brokerHost = "localhost";
		brokerPort = 9092;
		topic = "default_topic";
		partition = 0;
		factoryClass = "org.elasticsearch.river.kafka.JsonMessageHandlerFactory";
	}
    
    if (settings.settings().containsKey("index")) {
        Map<String, Object> indexSettings = (Map<String, Object>) settings.settings().get("index");
        bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size_bytes"), 10*1024*1024);
        if (indexSettings.containsKey("bulk_timeout")) {
            bulkTimeout = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get("bulk_timeout"), "10ms"), TimeValue.timeValueMillis(10000));
        } else {
            bulkTimeout = TimeValue.timeValueMillis(10);
        }
    } else {
        bulkSize = 10*1024*1024;
        bulkTimeout = TimeValue.timeValueMillis(10000);
    }
    
    if (settings.settings().containsKey("statsd")) {
        Map<String, Object> statsdSettings = (Map<String, Object>) settings.settings().get("statsd");
        statsdHost = (String)statsdSettings.get("host");
        statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get("port"), 8125);
        statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get("prefix"), "es-kafka-river");
    }
	else
	{
		statsdHost = null;
		statsdPort = -1;
		statsdPrefix = null;
	}
}
 

public void testIt()
{
	Map<String, Object> kafka = new HashMap<>();
	kafka.put("zookeeper", "zoo-host");
	kafka.put("broker_host", "broker-host");
	kafka.put("broker_port", "9999");
	kafka.put("topic", "my_topic");
	kafka.put("partition", "777");
	kafka.put("message_handler_factory_class", "my.factory.class.MyFactory");
	
	Map<String, Object> index = new HashMap<>();
	index.put("bulk_size_bytes", "1717171");
	index.put("bulk_timeout", "111ms");
	
	Map<String, Object> statsd = new HashMap<>();
	statsd.put("host", "some.host");
	statsd.put("port", "1234");
	statsd.put("prefix", "boo.yeah");
	
	Map<String, Object> map = new HashMap<>();
	map.put("kafka", kafka);
	map.put("index", index);
	map.put("statsd", statsd);
	
	RiverSettings settings = new RiverSettings(
		ImmutableSettings.settingsBuilder().build(), 
		map);
	KafkaRiverConfig c = new KafkaRiverConfig(settings);
	
	assertEquals("broker-host", c.brokerHost);
	assertEquals(9999, c.brokerPort);
	assertEquals("zoo-host", c.zookeeper);
	assertEquals("my.factory.class.MyFactory", c.factoryClass);
	assertEquals(1717171, c.bulkSize);
	assertEquals(777, c.partition);
	assertEquals("my_topic", c.topic);
	assertEquals(111, c.bulkTimeout.millis());
	
	
	assertEquals(1234, c.statsdPort);
	assertEquals("some.host", c.statsdHost);
	assertEquals("boo.yeah", c.statsdPrefix);
}
 

@Inject
@SuppressWarnings({ "unchecked" })
protected S3River(RiverName riverName, RiverSettings settings, Client client, ThreadPool threadPool) throws Exception{
   super(riverName, settings);
   this.client = client;
   this.threadPool = threadPool;
   this.riverStatus = RiverStatus.UNKNOWN;
   
   // Deal with connector settings.
   if (settings.settings().containsKey("amazon-s3")){
      Map<String, Object> feed = (Map<String, Object>)settings.settings().get("amazon-s3");
      
      // Retrieve feed settings.
      String feedname = XContentMapValues.nodeStringValue(feed.get("name"), null);
      String bucket = XContentMapValues.nodeStringValue(feed.get("bucket"), null);
      String pathPrefix = XContentMapValues.nodeStringValue(feed.get("pathPrefix"), null);
      String downloadHost = XContentMapValues.nodeStringValue(feed.get("download_host"), null);
      int updateRate = XContentMapValues.nodeIntegerValue(feed.get("update_rate"), 15 * 60 * 1000);
      boolean jsonSupport = XContentMapValues.nodeBooleanValue(feed.get("json_support"), false);
      double indexedCharsRatio  = XContentMapValues.nodeDoubleValue(feed.get("indexed_chars_ratio"), 0.0);
      
      String[] includes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.includes");
      String[] excludes = S3RiverUtil.buildArrayFromSettings(settings.settings(), "amazon-s3.excludes");
      
      // Retrieve connection settings.
      String accessKey = XContentMapValues.nodeStringValue(feed.get("accessKey"), null);
      String secretKey = XContentMapValues.nodeStringValue(feed.get("secretKey"), null);
      boolean useIAMRoleForEC2 = XContentMapValues.nodeBooleanValue(feed.get("use_EC2_IAM"), false);
      
      feedDefinition = new S3RiverFeedDefinition(feedname, bucket, pathPrefix, downloadHost,
            updateRate, Arrays.asList(includes), Arrays.asList(excludes), accessKey, secretKey, useIAMRoleForEC2,
            jsonSupport, indexedCharsRatio);
   } else {
      logger.error("You didn't define the amazon-s3 settings. Exiting... See https://github.com/lbroudoux/es-amazon-s3-river");
      indexName = null;
      typeName = null;
      bulkSize = 100;
      feedDefinition = null;
      s3 = null;
      return;
   }
   
   // Deal with index settings if provided.
   if (settings.settings().containsKey("index")) {
      Map<String, Object> indexSettings = (Map<String, Object>)settings.settings().get("index");
      
      indexName = XContentMapValues.nodeStringValue(indexSettings.get("index"), riverName.name());
      typeName = XContentMapValues.nodeStringValue(indexSettings.get("type"), S3RiverUtil.INDEX_TYPE_DOC);
      bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get("bulk_size"), 100);
   } else {
      indexName = riverName.name();
      typeName = S3RiverUtil.INDEX_TYPE_DOC;
      bulkSize = 100;
   }
   
   // We need to connect to Amazon S3 after ensure mandatory settings are here.
   if (feedDefinition.getBucket() == null){
      logger.error("Amazon S3 bucket should not be null. Please fix this.");
      throw new IllegalArgumentException("Amazon S3 bucket should not be null.");
   }
   // Connect using the appropriate authentication process.
   if (feedDefinition.getAccessKey() == null && feedDefinition.getSecretKey() == null) {
      s3 = new S3Connector(feedDefinition.isUseIAMRoleForEC2());
   } else {
      s3 = new S3Connector(feedDefinition.getAccessKey(), feedDefinition.getSecretKey());
   }
   try {
      s3.connectUserBucket(feedDefinition.getBucket(), feedDefinition.getPathPrefix());
   } catch (AmazonS3Exception ase){
      logger.error("Exception while connecting Amazon S3 user bucket. "
            + "Either access key, secret key, IAM Role or bucket name are incorrect");
      throw ase;
   }

   this.riverStatus = RiverStatus.INITIALIZED;
}
 

public RiverConfig(RiverName riverName, RiverSettings riverSettings) {

        // Extract kafka related configuration
        if (riverSettings.settings().containsKey("kafka")) {
            Map<String, Object> kafkaSettings = (Map<String, Object>) riverSettings.settings().get("kafka");

            topic = (String) kafkaSettings.get(TOPIC);
            zookeeperConnect = XContentMapValues.nodeStringValue(kafkaSettings.get(ZOOKEEPER_CONNECT), "localhost");
            zookeeperConnectionTimeout = XContentMapValues.nodeIntegerValue(kafkaSettings.get(ZOOKEEPER_CONNECTION_TIMEOUT), 10000);
            messageType = MessageType.fromValue(XContentMapValues.nodeStringValue(kafkaSettings.get(MESSAGE_TYPE),
                    MessageType.JSON.toValue()));
        } else {
            zookeeperConnect = "localhost";
            zookeeperConnectionTimeout = 10000;
            topic = "elasticsearch-river-kafka";
            messageType = MessageType.JSON;
        }

        // Extract ElasticSearch related configuration
        if (riverSettings.settings().containsKey("index")) {
            Map<String, Object> indexSettings = (Map<String, Object>) riverSettings.settings().get("index");
            indexName = XContentMapValues.nodeStringValue(indexSettings.get(INDEX_NAME), riverName.name());
            typeName = XContentMapValues.nodeStringValue(indexSettings.get(MAPPING_TYPE), "status");
            bulkSize = XContentMapValues.nodeIntegerValue(indexSettings.get(BULK_SIZE), 100);
            concurrentRequests = XContentMapValues.nodeIntegerValue(indexSettings.get(CONCURRENT_REQUESTS), 1);
            actionType = ActionType.fromValue(XContentMapValues.nodeStringValue(indexSettings.get(ACTION_TYPE),
                    ActionType.INDEX.toValue()));
            flushInterval = TimeValue.parseTimeValue(XContentMapValues.nodeStringValue(indexSettings.get(FLUSH_INTERVAL), "12h"), FLUSH_12H);
        } else {
            indexName = riverName.name();
            typeName = "status";
            bulkSize = 100;
            concurrentRequests = 1;
            actionType = ActionType.INDEX;
            flushInterval = FLUSH_12H;
        }
        
        // Extract StatsD related configuration
        if (riverSettings.settings().containsKey("statsd")) {
            Map<String, Object> statsdSettings = (Map<String, Object>) riverSettings.settings().get("statsd");
            statsdHost = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_HOST), "localhost");
            statsdPrefix = XContentMapValues.nodeStringValue(statsdSettings.get(STATSD_PREFIX), "kafka_river");
            statsdPort = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_PORT), 8125);
            statsdIntervalInSeconds = XContentMapValues.nodeIntegerValue(statsdSettings.get(STATSD_INTERVAL_IN_SECONDS), 10);
        }
    }