Java Code Examples for backtype.storm.Config#putAll()
The following examples show how to use
backtype.storm.Config#putAll() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FluxBuilder.java From flux with Apache License 2.0 | 6 votes |
private static StormTopology buildExternalTopology(ObjectDef def, ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { Object topologySource = buildObject(def, context); String methodName = context.getTopologyDef().getTopologySource().getMethodName(); Method getTopology = findGetTopologyMethod(topologySource, methodName); if(getTopology.getParameterTypes()[0].equals(Config.class)){ Config config = new Config(); config.putAll(context.getTopologyDef().getConfig()); return (StormTopology) getTopology.invoke(topologySource, config); } else { return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig()); } }
Example 2
Source File: LocalRunner.java From storm-benchmark with Apache License 2.0 | 6 votes |
private static void run(String name) throws ClassNotFoundException, IllegalAccessException, InstantiationException, AlreadyAliveException, InvalidTopologyException { LOG.info("running benchmark " + name); IBenchmark benchmark = (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name); Config config = new Config(); config.putAll(Utils.readStormConfig()); config.setDebug(true); StormTopology topology = benchmark.getTopology(config); LocalCluster localCluster = new LocalCluster(); localCluster.submitTopology(name, config, topology); final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME, MetricsCollectorConfig.DEFAULT_TOTAL_TIME); IMetricsCollector collector = benchmark.getMetricsCollector(config, topology); collector.run(); try { Thread.sleep(runtime); } catch (InterruptedException e) { LOG.error("benchmark interrupted", e); } localCluster.shutdown(); }
Example 3
Source File: FluxBuilder.java From jstorm with Apache License 2.0 | 6 votes |
private static StormTopology buildExternalTopology(ObjectDef def, ExecutionContext context) throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException, InvocationTargetException { Object topologySource = buildObject(def, context); String methodName = context.getTopologyDef().getTopologySource().getMethodName(); Method getTopology = findGetTopologyMethod(topologySource, methodName); if(getTopology.getParameterTypes()[0].equals(Config.class)){ Config config = new Config(); config.putAll(context.getTopologyDef().getConfig()); return (StormTopology) getTopology.invoke(topologySource, config); } else { return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig()); } }
Example 4
Source File: TestTridentTopology.java From jstorm with Apache License 2.0 | 6 votes |
public static void main(String[] args) throws Exception{ if (args.length == 0) { System.err.println("Please input configuration file"); System.exit(-1); } Map conf = LoadConfig.LoadConf(args[0]); if (conf == null) { LOG.error("Failed to load config"); } else { Config config = new Config(); config.putAll(conf); config.setMaxSpoutPending(10); config.put(LoadConfig.TOPOLOGY_TYPE, "Trident"); StormSubmitter.submitTopology("WordCount", config, buildTopology()); } }
Example 5
Source File: FluxBuilder.java From flux with Apache License 2.0 | 5 votes |
/** * Given a topology definition, return a populated `backtype.storm.Config` instance. * * @param topologyDef * @return */ public static Config buildConfig(TopologyDef topologyDef) { // merge contents of `config` into topology config Config conf = new Config(); conf.putAll(topologyDef.getConfig()); return conf; }
Example 6
Source File: KafkaProducer.java From storm-benchmark with Apache License 2.0 | 5 votes |
@Override public StormTopology getTopology(Config config) { config.putAll(getKafkaConfig(config)); final int spoutNum = BenchmarkUtils.getInt(config , SPOUT_NUM, DEFAULT_SPOUT_NUM); final int boltNum = BenchmarkUtils.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID); return builder.createTopology(); }
Example 7
Source File: JStormHelper.java From jstorm with Apache License 2.0 | 5 votes |
public static Config getConfig(String[] args) { Config ret = new Config(); if (args == null || args.length == 0) { return ret; } if (StringUtils.isBlank(args[0])) { return ret; } Map conf = JStormHelper.LoadConf(args[0]); ret.putAll(conf); return ret; }
Example 8
Source File: ReachTopology.java From jstorm with Apache License 2.0 | 5 votes |
public static void main(String[] args) throws Exception { LinearDRPCTopologyBuilder builder = construct(); Config conf = new Config(); conf.setNumWorkers(6); if (args.length != 0) { try { Map yamlConf = LoadConf.LoadYaml(args[0]); if (yamlConf != null) { conf.putAll(yamlConf); } } catch (Exception e) { System.out.println("Input " + args[0] + " isn't one yaml "); } StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, builder.createRemoteTopology()); } else { conf.setMaxTaskParallelism(3); LocalDRPC drpc = new LocalDRPC(); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, conf, builder.createLocalTopology(drpc)); JStormUtils.sleepMs(10000); String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" }; for (String url : urlsToTry) { System.out.println("Reach of " + url + ": " + drpc.execute(TOPOLOGY_NAME, url)); } cluster.shutdown(); drpc.shutdown(); } }
Example 9
Source File: FluxBuilder.java From jstorm with Apache License 2.0 | 5 votes |
/** * Given a topology definition, return a populated `org.apache.storm.Config` instance. * * @param topologyDef * @return */ public static Config buildConfig(TopologyDef topologyDef) { // merge contents of `config` into topology config Config conf = new Config(); conf.putAll(topologyDef.getConfig()); return conf; }
Example 10
Source File: Utils.java From jstorm with Apache License 2.0 | 5 votes |
public static Config getConfig(String[] args) { Config ret = new Config(); if (args == null || args.length == 0) { return ret; } if (StringUtils.isBlank(args[0])) { return ret; } Map conf = LoadConf(args[0]); ret.putAll(conf); return ret; }
Example 11
Source File: Utils.java From jstorm with Apache License 2.0 | 5 votes |
public static Config getConfig(String[] args) { Config ret = new Config(); if (args == null || args.length == 0) { return ret; } if (StringUtils.isBlank(args[0])) { return ret; } Map conf = LoadConf(args[0]); ret.putAll(conf); return ret; }
Example 12
Source File: StreamingApp.java From storm-solr with Apache License 2.0 | 4 votes |
public static Config getConfig(String env, File config) throws IOException { ConfigObject configObject = new ConfigSlurper(env).parse(readGroovyConfigScript(config)); Config stormConf = new Config(); Map flatten = configObject.flatten(); stormConf.putAll(flatten); Map<String, Class> dataTypes = new HashMap<String, Class>(); dataTypes.put("topology.workers", Integer.class); dataTypes.put("topology.acker.executors", Integer.class); dataTypes.put("topology.message.timeout.secs", Integer.class); dataTypes.put("topology.max.task.parallelism", Integer.class); dataTypes.put("topology.stats.sample.rate", Double.class); // this will convert built in properties as storm uses old school properties for (Field field : stormConf.getClass().getFields()) { if (Modifier.isStatic(field.getModifiers()) && Modifier.isPublic(field.getModifiers())) { String property = field.getName().toLowerCase().replace('_', '.'); if (property.startsWith("java.")) { // don't mess with Java system properties here continue; } Object override = flatten.get(property); if (override != null) { stormConf.put(property, override); System.out.println("Overrode property '" + property + "' with value [" + override + "] from Config.groovy of type " + override.getClass().getName()); } String system = System.getProperty(property, null); if (system != null) { if (dataTypes.containsKey(property)) { Class aClass = dataTypes.get(property); try { Method valueOf = aClass.getMethod("valueOf", String.class); stormConf.put(property, valueOf.invoke(aClass, system)); System.out.println("Overrode property '" + property + "' with value [" + stormConf.get(property) + "] from -D System property of type " + aClass.getName()); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } } else { stormConf.put(property, system); System.out.println("Overrode property '" + property + "' with String value [" + system + "] from -D System property"); } } } } return stormConf; }
Example 13
Source File: StormDoTask.java From incubator-samoa with Apache License 2.0 | 4 votes |
/** * The main method. * * @param args * the arguments */ public static void main(String[] args) { List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args)); boolean isLocal = isLocal(tmpArgs); int numWorker = StormSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); // convert the arguments into Storm topology StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); String topologyName = stormTopo.getTopologyName(); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); conf.setDebug(false); if (isLocal) { // local mode conf.setMaxTaskParallelism(numWorker); backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); backtype.storm.utils.Utils.sleep(600 * 1000); cluster.killTopology(topologyName); cluster.shutdown(); } else { // cluster mode conf.setNumWorkers(numWorker); try { backtype.storm.StormSubmitter.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); } catch (backtype.storm.generated.AlreadyAliveException ale) { ale.printStackTrace(); } catch (backtype.storm.generated.InvalidTopologyException ite) { ite.printStackTrace(); } } }
Example 14
Source File: StormTopologySubmitter.java From incubator-samoa with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws IOException { Properties props = StormSamoaUtils.getProperties(); String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); if (uploadedJarLocation == null) { logger.error("Invalid properties file. It must have key {}", StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); return; } List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args)); int numWorkers = StormSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); conf.putAll(Utils.readCommandLineOpts()); conf.setDebug(false); conf.setNumWorkers(numWorkers); String profilerOption = props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY); if (profilerOption != null) { String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); StringBuilder optionBuilder = new StringBuilder(); if (topoWorkerChildOpts != null) { optionBuilder.append(topoWorkerChildOpts); optionBuilder.append(' '); } optionBuilder.append(profilerOption); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString()); } Map<String, Object> myConfigMap = new HashMap<String, Object>(conf); StringWriter out = new StringWriter(); try { JSONValue.writeJSONString(myConfigMap, out); } catch (IOException e) { System.out.println("Error in writing JSONString"); e.printStackTrace(); return; } Config config = new Config(); config.putAll(Utils.readStormConfig()); NimbusClient nc = NimbusClient.getConfiguredClient(config); String topologyName = stormTopo.getTopologyName(); try { System.out.println("Submitting topology with name: " + topologyName); nc.getClient().submitTopology(topologyName, uploadedJarLocation, out.toString(), stormTopo.getStormBuilder().createTopology()); System.out.println(topologyName + " is successfully submitted"); } catch (AlreadyAliveException aae) { System.out.println("Fail to submit " + topologyName + "\nError message: " + aae.get_msg()); } catch (InvalidTopologyException ite) { System.out.println("Invalid topology for " + topologyName); ite.printStackTrace(); } catch (TException te) { System.out.println("Texception for " + topologyName); te.printStackTrace(); } }
Example 15
Source File: StormDoTask.java From samoa with Apache License 2.0 | 4 votes |
/** * The main method. * * @param args the arguments */ public static void main(String[] args) { List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args)); boolean isLocal = isLocal(tmpArgs); int numWorker = StormSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); //convert the arguments into Storm topology StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); String topologyName = stormTopo.getTopologyName(); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); conf.setDebug(false); if(isLocal){ //local mode conf.setMaxTaskParallelism(numWorker); backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster(); cluster.submitTopology(topologyName , conf, stormTopo.getStormBuilder().createTopology()); backtype.storm.utils.Utils.sleep(600*1000); cluster.killTopology(topologyName); cluster.shutdown(); }else{ //cluster mode conf.setNumWorkers(numWorker); try { backtype.storm.StormSubmitter.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology()); } catch (backtype.storm.generated.AlreadyAliveException ale) { ale.printStackTrace(); } catch (backtype.storm.generated.InvalidTopologyException ite) { ite.printStackTrace(); } } }
Example 16
Source File: StormTopologySubmitter.java From samoa with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws IOException{ Properties props = StormSamoaUtils.getProperties(); String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); if(uploadedJarLocation == null){ logger.error("Invalid properties file. It must have key {}", StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY); return; } List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args)); int numWorkers = StormSamoaUtils.numWorkers(tmpArgs); args = tmpArgs.toArray(new String[0]); StormTopology stormTopo = StormSamoaUtils.argsToTopology(args); Config conf = new Config(); conf.putAll(Utils.readStormConfig()); conf.putAll(Utils.readCommandLineOpts()); conf.setDebug(false); conf.setNumWorkers(numWorkers); String profilerOption = props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY); if(profilerOption != null){ String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS); StringBuilder optionBuilder = new StringBuilder(); if(topoWorkerChildOpts != null){ optionBuilder.append(topoWorkerChildOpts); optionBuilder.append(' '); } optionBuilder.append(profilerOption); conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString()); } Map<String, Object> myConfigMap = new HashMap<String, Object>(conf); StringWriter out = new StringWriter(); try { JSONValue.writeJSONString(myConfigMap, out); } catch (IOException e) { System.out.println("Error in writing JSONString"); e.printStackTrace(); return; } Config config = new Config(); config.putAll(Utils.readStormConfig()); String nimbusHost = (String) config.get(Config.NIMBUS_HOST); NimbusClient nc = new NimbusClient(nimbusHost); String topologyName = stormTopo.getTopologyName(); try { System.out.println("Submitting topology with name: " + topologyName); nc.getClient().submitTopology(topologyName, uploadedJarLocation, out.toString(), stormTopo.getStormBuilder().createTopology()); System.out.println(topologyName + " is successfully submitted"); } catch (AlreadyAliveException aae) { System.out.println("Fail to submit " + topologyName + "\nError message: " + aae.get_msg()); } catch (InvalidTopologyException ite) { System.out.println("Invalid topology for " + topologyName); ite.printStackTrace(); } catch (TException te) { System.out.println("Texception for " + topologyName); te.printStackTrace(); } }