Java Code Examples for backtype.storm.Config.putAll()

The following are Jave code examples for showing how to use putAll() of the backtype.storm.Config class. You can vote up the examples you like. Your votes will be used in our system to get more good examples.
Example 1
Project: web-crawler   File: WebCrawlerTopology.java   Source Code and License Vote up 6 votes
public static void main(String[] args) throws Exception {

        if(args.length < 1){
            System.err.println("[ERROR] Configuration File Required");
        }
        Config conf = new Config();

        // Store all the configuration in the Storm conf object
        conf.putAll(readConfigFile(args[0]));

        //Second arg should be local in order to run locally
        if(args.length  < 2 || (args.length  == 2 && !args[1].equals("local"))) {
            StormSubmitter.submitTopologyWithProgressBar("crawler_topology", conf, buildTopology(conf, null));
        }
        else {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster localcluster = new LocalCluster();
            localcluster.submitTopology("crawler_topology",conf,buildTopology(conf, drpc));

            String searchQuery = "elasticsearch";
            System.out.println("---* Result (search): " + drpc.execute("search",  searchQuery));
        }
    }
 
Example 2
Project: alfresco-apache-storm-demo   File: ConfUtils.java   Source Code and License Vote up 5 votes
public static Config loadConf(String resource) throws FileNotFoundException {
    Config conf = new Config();
    Yaml yaml = new Yaml();
    Map ret = (Map) yaml.load(new InputStreamReader(new FileInputStream(
            resource), Charset.defaultCharset()));
    if (ret == null) {
        ret = new HashMap();
    }
    conf.putAll(ret);
    return conf;
}
 
Example 3
Project: jstrom   File: ReachTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
	
    LinearDRPCTopologyBuilder builder = construct();
  
    Config conf = new Config();
    conf.setNumWorkers(6);
    if (args.length != 0) {
    	
    	try {
     	Map yamlConf = LoadConf.LoadYaml(args[0]);
     	if (yamlConf != null) {
     		conf.putAll(yamlConf);
     	}
    	}catch (Exception e) {
    		System.out.println("Input " + args[0] + " isn't one yaml ");
    	}
    	
    	
    	StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, builder.createRemoteTopology());
    }else {
    
    
        conf.setMaxTaskParallelism(3);
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, conf, builder.createLocalTopology(drpc));
        
        JStormUtils.sleepMs(50000);
        
        String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com"};
        for(String url: urlsToTry) {
            System.out.println("Reach of " + url + ": " + drpc.execute(TOPOLOGY_NAME, url));
        }
        
        cluster.shutdown();
        drpc.shutdown();
    }
}
 
Example 4
Project: jstrom   File: TransactionalGlobalCount.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {
  MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
  TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("global-count", "spout", spout, 3);
  builder.setBolt("partial-count", new BatchCount(), 5).noneGrouping("spout");
  builder.setBolt("sum", new UpdateGlobalCount()).globalGrouping("partial-count");

  Config config = new Config();
  config.setDebug(true);
  config.setMaxSpoutPending(3);
  
  if (args.length == 0){
      LocalCluster cluster = new LocalCluster();    
  
      cluster.submitTopology("global-count-topology", config, builder.buildTopology());
  
      Thread.sleep(100000);
      cluster.shutdown();
  }else {
  	
      config.setNumWorkers(3);
  	try {
      	Map yamlConf = LoadConf.LoadYaml(args[0]);
      	if (yamlConf != null) {
      		config.putAll(yamlConf);
      	}
  	}catch (Exception e) {
  		System.out.println("Input " + args[0] + " isn't one yaml ");
  	}

      StormSubmitter.submitTopology("global", config, builder.buildTopology());
  }
}
 
Example 5
Project: realtime-event-processing   File: URLEventProcessingTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception {

        if(args.length != 2){
            System.err.println("[ERROR] Configuration File Required");
        }
        Config conf = new Config();

        //Map topologyConfig = readConfigFile(args[0]);
        //conf.putAll(topologyConfig);

        // Store all the configuration in the Storm conf object
        conf.putAll(ConfigReader.readConfigFile(args[0]));

        //Second arg should be local in order to run locally
        if(args[1].equals("local"))
        {
            LocalDRPC drpc = new LocalDRPC();
            LocalCluster localcluster = new LocalCluster();
            localcluster.submitTopology("url_event_processing",conf,buildTopology(conf, drpc));

            String searchQuery = "HoloLens crawl_test";
            System.out.println("---* Result: " + drpc.execute("search",  searchQuery));
        }
        else
        {
            StormSubmitter.submitTopologyWithProgressBar("url_event_processing", conf, buildTopology(conf, null));
        }
    }
 
Example 6
Project: realtime-event-processing   File: DocEventProcessingTopology.java   Source Code and License Vote up 5 votes
public static void main(String[] args) throws Exception{
    if(args.length != 2){
        System.err.println("[ERROR] Configuration File Required");
    }
    Config conf = new Config();

    //Map topologyConfig = readConfigFile(args[0]);
    //conf.putAll(topologyConfig);

    // Store all the configuration in the Storm conf object
    conf.putAll(ConfigReader.readConfigFile(args[0]));

    //Second arg should be local in order to run locally
    if(args[1].equals("local"))
    {
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster localcluster = new LocalCluster();
        localcluster.submitTopology("doc_event_processing",conf,buildTopology(conf, drpc));

        String searchQuery = "HoloLens crawl_test";
        System.out.println("---* Result: " + drpc.execute("search",  searchQuery));
    }
    else
    {
        StormSubmitter.submitTopologyWithProgressBar("doc_event_processing", conf, buildTopology(conf, null));
    }
}
 
Example 7
Project: alfresco-apache-storm-demo   File: FetcherBolt.java   Source Code and License Vote up 4 votes
@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collector) {

    _collector = collector;

    Config conf = new Config();
    conf.putAll(stormConf);

    checkConfiguration(conf);

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss",
            Locale.ENGLISH);
    long start = System.currentTimeMillis();
    LOG.info("[Fetcher #{}] : starting at {}", taskIndex, sdf.format(start));

    // Register a "MultiCountMetric" to count different events in this bolt
    // Storm will emit the counts every n seconds to a special bolt via a
    // system stream
    // The data can be accessed by registering a "MetricConsumer" in the
    // topology
    this.eventCounter = context.registerMetric("fetcher_counter",
            new MultiCountMetric(), 10);

    this.metricGauge = context.registerMetric("fetcher",
            new MultiCountMetric(), 10);

    this.averagedMetrics = context.registerMetric("fetcher_average",
            new MultiReducedMetric(new MeanReducer()), 10);

    protocolFactory = new ProtocolFactory(conf);

    this.fetchQueues = new FetchItemQueues(conf);

    this.taskIndex = context.getThisTaskIndex();

    int threadCount = ConfUtils.getInt(conf, "fetcher.threads.number", 10);
    for (int i = 0; i < threadCount; i++) { // spawn threads
        new FetcherThread(conf).start();
    }

    String urlconfigfile = ConfUtils.getString(conf,
            "urlfilters.config.file", "urlfilters.json");

    if (urlconfigfile != null)
        try {
            urlFilters = new URLFilters(conf, urlconfigfile);
        } catch (IOException e) {
            LOG.error("Exception caught while loading the URLFilters");
            throw new RuntimeException(
                    "Exception caught while loading the URLFilters", e);
        }

    allowRedirs = ConfUtils.getBoolean(stormConf,
            com.digitalpebble.storm.crawler.Constants.AllowRedirParamName,
            true);

    metadataTransfer = MetadataTransfer.getInstance(stormConf);
}
 
Example 8
Project: incubator-samoa   File: StormDoTask.java   Source Code and License Vote up 4 votes
/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  boolean isLocal = isLocal(tmpArgs);
  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  if (isLocal) {
    // local mode
    conf.setMaxTaskParallelism(numWorker);

    backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
    cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

    backtype.storm.utils.Utils.sleep(600 * 1000);

    cluster.killTopology(topologyName);
    cluster.shutdown();

  } else {
    // cluster mode
    conf.setNumWorkers(numWorker);
    try {
      backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
          stormTopo.getStormBuilder().createTopology());
    } catch (backtype.storm.generated.AlreadyAliveException ale) {
      ale.printStackTrace();
    } catch (backtype.storm.generated.InvalidTopologyException ite) {
      ite.printStackTrace();
    }
  }
}
 
Example 9
Project: incubator-samoa   File: StormTopologySubmitter.java   Source Code and License Vote up 4 votes
public static void main(String[] args) throws IOException {
  Properties props = StormSamoaUtils.getProperties();

  String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
  if (uploadedJarLocation == null) {
    logger.error("Invalid properties file. It must have key {}",
        StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
    return;
  }

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
  int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.putAll(Utils.readCommandLineOpts());
  conf.setDebug(false);
  conf.setNumWorkers(numWorkers);

  String profilerOption =
      props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
  if (profilerOption != null) {
    String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
    StringBuilder optionBuilder = new StringBuilder();
    if (topoWorkerChildOpts != null) {
      optionBuilder.append(topoWorkerChildOpts);
      optionBuilder.append(' ');
    }
    optionBuilder.append(profilerOption);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
  }

  Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
  StringWriter out = new StringWriter();

  try {
    JSONValue.writeJSONString(myConfigMap, out);
  } catch (IOException e) {
    System.out.println("Error in writing JSONString");
    e.printStackTrace();
    return;
  }

  Config config = new Config();
  config.putAll(Utils.readStormConfig());

  NimbusClient nc = NimbusClient.getConfiguredClient(config);
  String topologyName = stormTopo.getTopologyName();
  try {
    System.out.println("Submitting topology with name: "
        + topologyName);
    nc.getClient().submitTopology(topologyName, uploadedJarLocation,
        out.toString(), stormTopo.getStormBuilder().createTopology());
    System.out.println(topologyName + " is successfully submitted");

  } catch (AlreadyAliveException aae) {
    System.out.println("Fail to submit " + topologyName
        + "\nError message: " + aae.get_msg());
  } catch (InvalidTopologyException ite) {
    System.out.println("Invalid topology for " + topologyName);
    ite.printStackTrace();
  } catch (TException te) {
    System.out.println("Texception for " + topologyName);
    te.printStackTrace();
  }
}
 
Example 10
Project: storm-solr   File: StreamingApp.java   Source Code and License Vote up 3 votes
public static Config getConfig(String env, File config) throws IOException {
  ConfigObject configObject = new ConfigSlurper(env).parse(readGroovyConfigScript(config));
  Config stormConf = new Config();
  Map flatten = configObject.flatten();
  stormConf.putAll(flatten);

  Map<String, Class> dataTypes = new HashMap<String, Class>();
  dataTypes.put("topology.workers", Integer.class);
  dataTypes.put("topology.acker.executors", Integer.class);
  dataTypes.put("topology.message.timeout.secs", Integer.class);
  dataTypes.put("topology.max.task.parallelism", Integer.class);
  dataTypes.put("topology.stats.sample.rate", Double.class);

  // this will convert built in properties as storm uses old school properties
  for (Field field : stormConf.getClass().getFields()) {
    if (Modifier.isStatic(field.getModifiers())
      && Modifier.isPublic(field.getModifiers())) {
      String property = field.getName().toLowerCase().replace('_', '.');
      if (property.startsWith("java.")) {
        // don't mess with Java system properties here
        continue;
      }

      Object override = flatten.get(property);
      if (override != null) {
        stormConf.put(property, override);
        System.out.println("Overrode property '" + property + "' with value [" + override + "] from Config.groovy of type " + override.getClass().getName());
      }
      String system = System.getProperty(property, null);
      if (system != null) {
        if (dataTypes.containsKey(property)) {
          Class aClass = dataTypes.get(property);
          try {
            Method valueOf = aClass.getMethod("valueOf", String.class);
            stormConf.put(property, valueOf.invoke(aClass, system));
            System.out.println("Overrode property '" + property + "' with value [" + stormConf.get(property) + "] from -D System property of type " + aClass.getName());
          } catch (Exception e) {
            throw new RuntimeException(e.getMessage(), e);
          }
        } else {
          stormConf.put(property, system);
          System.out.println("Overrode property '" + property + "' with String value [" + system + "] from -D System property");
        }
      }
    }
  }

  return stormConf;
}