Java Code Examples for backtype.storm.Config#putAll()

The following examples show how to use backtype.storm.Config#putAll() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FluxBuilder.java    From flux with Apache License 2.0 6 votes vote down vote up 
private static StormTopology buildExternalTopology(ObjectDef def, ExecutionContext context)
        throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException,
        InvocationTargetException {

    Object topologySource = buildObject(def, context);

    String methodName = context.getTopologyDef().getTopologySource().getMethodName();
    Method getTopology = findGetTopologyMethod(topologySource, methodName);
    if(getTopology.getParameterTypes()[0].equals(Config.class)){
        Config config = new Config();
        config.putAll(context.getTopologyDef().getConfig());
        return (StormTopology) getTopology.invoke(topologySource, config);
    } else {
        return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig());
    }
}
Example 2
Source File: LocalRunner.java    From storm-benchmark with Apache License 2.0 6 votes vote down vote up 
private static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  LOG.info("running benchmark " + name);
  IBenchmark benchmark =  (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name);
  Config config = new Config();
  config.putAll(Utils.readStormConfig());
  config.setDebug(true);
  StormTopology topology = benchmark.getTopology(config);
  LocalCluster localCluster = new LocalCluster();
  localCluster.submitTopology(name, config, topology);
  final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME,
          MetricsCollectorConfig.DEFAULT_TOTAL_TIME);
  IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
  collector.run();
  try {
    Thread.sleep(runtime);
  } catch (InterruptedException e) {
    LOG.error("benchmark interrupted", e);
  }
  localCluster.shutdown();
}
Example 3
Source File: FluxBuilder.java    From jstorm with Apache License 2.0 6 votes vote down vote up 
private static StormTopology buildExternalTopology(ObjectDef def, ExecutionContext context)
        throws ClassNotFoundException, IllegalAccessException, InstantiationException, NoSuchMethodException,
        InvocationTargetException {

    Object topologySource = buildObject(def, context);

    String methodName = context.getTopologyDef().getTopologySource().getMethodName();
    Method getTopology = findGetTopologyMethod(topologySource, methodName);
    if(getTopology.getParameterTypes()[0].equals(Config.class)){
        Config config = new Config();
        config.putAll(context.getTopologyDef().getConfig());
        return (StormTopology) getTopology.invoke(topologySource, config);
    } else {
        return (StormTopology) getTopology.invoke(topologySource, context.getTopologyDef().getConfig());
    }
}
Example 4
Source File: TestTridentTopology.java    From jstorm with Apache License 2.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception{
	if (args.length == 0) {
		System.err.println("Please input configuration file");
		System.exit(-1);
	}

	Map conf = LoadConfig.LoadConf(args[0]);	

	if (conf == null) {
		LOG.error("Failed to load config");
	} else {
		Config config = new Config();
		config.putAll(conf);
        config.setMaxSpoutPending(10);
        config.put(LoadConfig.TOPOLOGY_TYPE, "Trident");
        StormSubmitter.submitTopology("WordCount", config, buildTopology());
	}
}
Example 5
Source File: FluxBuilder.java    From flux with Apache License 2.0 5 votes vote down vote up 
/**
 * Given a topology definition, return a populated `backtype.storm.Config` instance.
 *
 * @param topologyDef
 * @return
 */
public static Config buildConfig(TopologyDef topologyDef) {
    // merge contents of `config` into topology config
    Config conf = new Config();
    conf.putAll(topologyDef.getConfig());
    return conf;
}
Example 6
Source File: KafkaProducer.java    From storm-benchmark with Apache License 2.0 5 votes vote down vote up 
@Override
public StormTopology getTopology(Config config) {
  config.putAll(getKafkaConfig(config));

  final int spoutNum = BenchmarkUtils.getInt(config , SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int boltNum = BenchmarkUtils.getInt(config, BOLT_NUM, DEFAULT_BOLT_NUM);

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(BOLT_ID, bolt, boltNum).localOrShuffleGrouping(SPOUT_ID);
  return builder.createTopology();
}
Example 7
Source File: JStormHelper.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public static Config getConfig(String[] args) {
    Config ret = new Config();
    if (args == null || args.length == 0) {
        return ret;
    }

    if (StringUtils.isBlank(args[0])) {
        return ret;
    }

    Map conf = JStormHelper.LoadConf(args[0]);
    ret.putAll(conf);
    return ret;
}
Example 8
Source File: ReachTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
    
    LinearDRPCTopologyBuilder builder = construct();
    
    Config conf = new Config();
    conf.setNumWorkers(6);
    if (args.length != 0) {
        
        try {
            Map yamlConf = LoadConf.LoadYaml(args[0]);
            if (yamlConf != null) {
                conf.putAll(yamlConf);
            }
        } catch (Exception e) {
            System.out.println("Input " + args[0] + " isn't one yaml ");
        }
        
        StormSubmitter.submitTopology(TOPOLOGY_NAME, conf, builder.createRemoteTopology());
    } else {
        
        conf.setMaxTaskParallelism(3);
        LocalDRPC drpc = new LocalDRPC();
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(TOPOLOGY_NAME, conf, builder.createLocalTopology(drpc));
        
        JStormUtils.sleepMs(10000);
        
        String[] urlsToTry = new String[] { "foo.com/blog/1", "engineering.twitter.com/blog/5", "notaurl.com" };
        for (String url : urlsToTry) {
            System.out.println("Reach of " + url + ": " + drpc.execute(TOPOLOGY_NAME, url));
        }
        
        cluster.shutdown();
        drpc.shutdown();
    }
}
Example 9
Source File: FluxBuilder.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
/**
 * Given a topology definition, return a populated `org.apache.storm.Config` instance.
 *
 * @param topologyDef
 * @return
 */
public static Config buildConfig(TopologyDef topologyDef) {
    // merge contents of `config` into topology config
    Config conf = new Config();
    conf.putAll(topologyDef.getConfig());
    return conf;
}
Example 10
Source File: Utils.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public static Config getConfig(String[] args) {
    Config ret = new Config();
    if (args == null || args.length == 0) {
        return ret;
    }
    
    if (StringUtils.isBlank(args[0])) {
        return ret;
    }
    
    Map conf = LoadConf(args[0]);
    ret.putAll(conf);
    return ret;
}
Example 11
Source File: Utils.java    From jstorm with Apache License 2.0 5 votes vote down vote up 
public static Config getConfig(String[] args) {
    Config ret = new Config();
    if (args == null || args.length == 0) {
        return ret;
    }
    
    if (StringUtils.isBlank(args[0])) {
        return ret;
    }
    
    Map conf = LoadConf(args[0]);
    ret.putAll(conf);
    return ret;
}
Example 12
Source File: StreamingApp.java    From storm-solr with Apache License 2.0 4 votes vote down vote up 
public static Config getConfig(String env, File config) throws IOException {
  ConfigObject configObject = new ConfigSlurper(env).parse(readGroovyConfigScript(config));
  Config stormConf = new Config();
  Map flatten = configObject.flatten();
  stormConf.putAll(flatten);

  Map<String, Class> dataTypes = new HashMap<String, Class>();
  dataTypes.put("topology.workers", Integer.class);
  dataTypes.put("topology.acker.executors", Integer.class);
  dataTypes.put("topology.message.timeout.secs", Integer.class);
  dataTypes.put("topology.max.task.parallelism", Integer.class);
  dataTypes.put("topology.stats.sample.rate", Double.class);

  // this will convert built in properties as storm uses old school properties
  for (Field field : stormConf.getClass().getFields()) {
    if (Modifier.isStatic(field.getModifiers())
      && Modifier.isPublic(field.getModifiers())) {
      String property = field.getName().toLowerCase().replace('_', '.');
      if (property.startsWith("java.")) {
        // don't mess with Java system properties here
        continue;
      }

      Object override = flatten.get(property);
      if (override != null) {
        stormConf.put(property, override);
        System.out.println("Overrode property '" + property + "' with value [" + override + "] from Config.groovy of type " + override.getClass().getName());
      }
      String system = System.getProperty(property, null);
      if (system != null) {
        if (dataTypes.containsKey(property)) {
          Class aClass = dataTypes.get(property);
          try {
            Method valueOf = aClass.getMethod("valueOf", String.class);
            stormConf.put(property, valueOf.invoke(aClass, system));
            System.out.println("Overrode property '" + property + "' with value [" + stormConf.get(property) + "] from -D System property of type " + aClass.getName());
          } catch (Exception e) {
            throw new RuntimeException(e.getMessage(), e);
          }
        } else {
          stormConf.put(property, system);
          System.out.println("Overrode property '" + property + "' with String value [" + system + "] from -D System property");
        }
      }
    }
  }

  return stormConf;
}
Example 13
Source File: StormDoTask.java    From incubator-samoa with Apache License 2.0 4 votes vote down vote up 
/**
 * The main method.
 * 
 * @param args
 *          the arguments
 */
public static void main(String[] args) {

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));

  boolean isLocal = isLocal(tmpArgs);
  int numWorker = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);

  // convert the arguments into Storm topology
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
  String topologyName = stormTopo.getTopologyName();

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.setDebug(false);

  if (isLocal) {
    // local mode
    conf.setMaxTaskParallelism(numWorker);

    backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
    cluster.submitTopology(topologyName, conf, stormTopo.getStormBuilder().createTopology());

    backtype.storm.utils.Utils.sleep(600 * 1000);

    cluster.killTopology(topologyName);
    cluster.shutdown();

  } else {
    // cluster mode
    conf.setNumWorkers(numWorker);
    try {
      backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
          stormTopo.getStormBuilder().createTopology());
    } catch (backtype.storm.generated.AlreadyAliveException ale) {
      ale.printStackTrace();
    } catch (backtype.storm.generated.InvalidTopologyException ite) {
      ite.printStackTrace();
    }
  }
}
Example 14
Source File: StormTopologySubmitter.java    From incubator-samoa with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws IOException {
  Properties props = StormSamoaUtils.getProperties();

  String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
  if (uploadedJarLocation == null) {
    logger.error("Invalid properties file. It must have key {}",
        StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
    return;
  }

  List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
  int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);

  args = tmpArgs.toArray(new String[0]);
  StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

  Config conf = new Config();
  conf.putAll(Utils.readStormConfig());
  conf.putAll(Utils.readCommandLineOpts());
  conf.setDebug(false);
  conf.setNumWorkers(numWorkers);

  String profilerOption =
      props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
  if (profilerOption != null) {
    String topoWorkerChildOpts = (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
    StringBuilder optionBuilder = new StringBuilder();
    if (topoWorkerChildOpts != null) {
      optionBuilder.append(topoWorkerChildOpts);
      optionBuilder.append(' ');
    }
    optionBuilder.append(profilerOption);
    conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
  }

  Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
  StringWriter out = new StringWriter();

  try {
    JSONValue.writeJSONString(myConfigMap, out);
  } catch (IOException e) {
    System.out.println("Error in writing JSONString");
    e.printStackTrace();
    return;
  }

  Config config = new Config();
  config.putAll(Utils.readStormConfig());

  NimbusClient nc = NimbusClient.getConfiguredClient(config);
  String topologyName = stormTopo.getTopologyName();
  try {
    System.out.println("Submitting topology with name: "
        + topologyName);
    nc.getClient().submitTopology(topologyName, uploadedJarLocation,
        out.toString(), stormTopo.getStormBuilder().createTopology());
    System.out.println(topologyName + " is successfully submitted");

  } catch (AlreadyAliveException aae) {
    System.out.println("Fail to submit " + topologyName
        + "\nError message: " + aae.get_msg());
  } catch (InvalidTopologyException ite) {
    System.out.println("Invalid topology for " + topologyName);
    ite.printStackTrace();
  } catch (TException te) {
    System.out.println("Texception for " + topologyName);
    te.printStackTrace();
  }
}
Example 15
Source File: StormDoTask.java    From samoa with Apache License 2.0 4 votes vote down vote up 
/**
 * The main method.
 * 
 * @param args the arguments
 */
public static void main(String[] args) {

	 List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	
	boolean isLocal = isLocal(tmpArgs);
	int numWorker = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	
	//convert the arguments into Storm topology
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);
	String topologyName = stormTopo.getTopologyName();
	
   	Config conf = new Config();
   	conf.putAll(Utils.readStormConfig());
   	conf.setDebug(false);
   			
   	
	if(isLocal){
		//local mode
		conf.setMaxTaskParallelism(numWorker);
		
		backtype.storm.LocalCluster cluster = new backtype.storm.LocalCluster();
		cluster.submitTopology(topologyName , conf, stormTopo.getStormBuilder().createTopology());
		
		backtype.storm.utils.Utils.sleep(600*1000);
		
		cluster.killTopology(topologyName);
		cluster.shutdown();
		
	}else{
		//cluster mode
		conf.setNumWorkers(numWorker);
		try {
			backtype.storm.StormSubmitter.submitTopology(topologyName, conf,
					stormTopo.getStormBuilder().createTopology());
		} catch (backtype.storm.generated.AlreadyAliveException ale) {
			ale.printStackTrace();
		} catch (backtype.storm.generated.InvalidTopologyException ite) {
			ite.printStackTrace();
		}
	}
}
Example 16
Source File: StormTopologySubmitter.java    From samoa with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws IOException{
	Properties props = StormSamoaUtils.getProperties();
	
	String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
	if(uploadedJarLocation == null){
		logger.error("Invalid properties file. It must have key {}", 
				StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
		return;
	}
	
	List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

	Config conf = new Config();
	conf.putAll(Utils.readStormConfig());
	conf.putAll(Utils.readCommandLineOpts());
	conf.setDebug(false);
	conf.setNumWorkers(numWorkers);
	
	String profilerOption = 
			props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
	if(profilerOption != null){
		String topoWorkerChildOpts =  (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
		StringBuilder optionBuilder = new StringBuilder();
		if(topoWorkerChildOpts != null){
			optionBuilder.append(topoWorkerChildOpts);	
			optionBuilder.append(' ');
		}
		optionBuilder.append(profilerOption);
		conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
	}

	Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
	StringWriter out = new StringWriter();

	try {
		JSONValue.writeJSONString(myConfigMap, out);
	} catch (IOException e) {
		System.out.println("Error in writing JSONString");
		e.printStackTrace();
		return;
	}
	
	Config config = new Config();
	config.putAll(Utils.readStormConfig());
	
	String nimbusHost = (String) config.get(Config.NIMBUS_HOST);
			
	NimbusClient nc = new NimbusClient(nimbusHost);
	String topologyName = stormTopo.getTopologyName();
	try {
		System.out.println("Submitting topology with name: " 
				+ topologyName);
		nc.getClient().submitTopology(topologyName, uploadedJarLocation,
				out.toString(), stormTopo.getStormBuilder().createTopology());
		System.out.println(topologyName + " is successfully submitted");

	} catch (AlreadyAliveException aae) {
		System.out.println("Fail to submit " + topologyName
				+ "\nError message: " + aae.get_msg());
	} catch (InvalidTopologyException ite) {
		System.out.println("Invalid topology for " + topologyName);
		ite.printStackTrace();
	} catch (TException te) {
		System.out.println("Texception for " + topologyName);
		te.printStackTrace();
	} 		
}