backtype.storm.generated.InvalidTopologyException Java Examples

The following examples show how to use backtype.storm.generated.InvalidTopologyException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #2

Source File: LocalCluster.java From incubator-heron with Apache License 2.0

6 votes

@Override
@SuppressWarnings({"rawtypes", "unchecked"})
public void submitTopology(String topoName,
                           Map config,
                           StormTopology stormTopology)
    throws AlreadyAliveException, InvalidTopologyException {
  assertNotAlive();

  this.topologyName = topoName;
  this.conf = config;
  this.topology = stormTopology;

  simulator.submitTopology(topoName,
      ConfigUtils.translateConfig(config),
      stormTopology.getStormTopology());
}

Example #3

Source File: SequenceTopologyTool.java From jstorm with Apache License 2.0

6 votes

public void SetRemoteTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    Config conf = getConf();
    StormTopology topology = buildTopology();
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    if (streamName.contains("zeromq")) {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.zeroMq.MQContext");
        
    } else {
        conf.put(Config.STORM_MESSAGING_TRANSPORT, "com.alibaba.jstorm.message.netty.NettyContext");
    }
    
    StormSubmitter.submitTopology(streamName, conf, topology);
    
}

Example #4

Source File: WordCountTopology.java From incubator-heron with Apache License 2.0

6 votes

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
  if (args.length < 1) {
    throw new RuntimeException("Specify topology name");
  }

  int parallelism = 1;
  if (args.length > 1) {
    parallelism = Integer.parseInt(args[1]);
  }
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout("word", new WordSpout(), parallelism);
  builder.setBolt("consumer", new ConsumerBolt(), parallelism)
      .fieldsGrouping("word", new Fields("word"));
  Config conf = new Config();
  conf.setNumWorkers(parallelism);

  StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example #5

Source File: ServiceHandler.java From jstorm with Apache License 2.0

6 votes

/**
 * generate TaskInfo for every bolt or spout in ZK /ZK/tasks/topoologyId/xxx
 */
public void setupZkTaskInfo(Map<Object, Object> conf, String topologyId, StormClusterState stormClusterState) throws Exception {
    Map<Integer, TaskInfo> taskToTaskInfo = mkTaskComponentAssignments(conf, topologyId);

    // mkdir /ZK/taskbeats/topoologyId
    int masterId = NimbusUtils.getTopologyMasterId(taskToTaskInfo);
    TopologyTaskHbInfo topoTaskHbInfo = new TopologyTaskHbInfo(topologyId, masterId);
    data.getTasksHeartbeat().put(topologyId, topoTaskHbInfo);
    stormClusterState.topology_heartbeat(topologyId, topoTaskHbInfo);

    if (taskToTaskInfo == null || taskToTaskInfo.size() == 0) {
        throw new InvalidTopologyException("Failed to generate TaskIDs map");
    }
    // key is task id, value is task info
    stormClusterState.set_task(topologyId, taskToTaskInfo);
}

Example #6

Source File: BatchAckerTest.java From jstorm with Apache License 2.0

6 votes

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Config conf = JStormHelper.getConfig(args);
    int spoutParallelism = JStormUtils.parseInt(conf.get(SPOUT_PARALLELISM_HINT), 1);
    int splitParallelism = JStormUtils.parseInt(conf.get(SPLIT_PARALLELISM_HINT), 2);
    int countParallelism = JStormUtils.parseInt(conf.get(COUNT_PARALLELISM_HINT), 2);
    boolean isValueSpout = JStormUtils.parseBoolean(conf.get("is.value.spout"), false);

    TransactionTopologyBuilder builder = new TransactionTopologyBuilder();
    if (isValueSpout)
        builder.setSpoutWithAck("spout", new BatchAckerValueSpout(), spoutParallelism);
    else
        builder.setSpoutWithAck("spout", new BatchAckerSpout(), spoutParallelism);
    builder.setBoltWithAck("split", new BatchAckerSplit(), splitParallelism).localOrShuffleGrouping("spout");;
    builder.setBoltWithAck("count", new BatchAckerCount(), countParallelism).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}

Example #7

Source File: RocksDbHdfsState.java From jstorm with Apache License 2.0

6 votes

@Override
public void init(TopologyContext context) {
    try {
        this.topologyName = Common.topologyIdToName(context.getTopologyId());
    } catch (InvalidTopologyException e) {
        LOG.error("Failed get topology name by id-{}", context.getTopologyId());
        throw new RuntimeException(e.getMessage());
    }
    String workerDir = context.getWorkerIdDir();
    metricClient = new MetricClient(context);
    hdfsWriteLatency = metricClient.registerHistogram("HDFS write latency");
    hdfsDeleteLatency = metricClient.registerHistogram("HDFS delete latency");
    rocksDbFlushAndCpLatency = metricClient.registerHistogram("RocksDB flush and checkpoint latency");
    cleanPeriod = ConfigExtension.getTransactionBatchSnapshotTimeout(context.getStormConf()) * 5 * 1000;
    serializer = SerializerFactory.createSerailzer(context.getStormConf());
    initEnv(topologyName, context.getStormConf(), workerDir);
}

Example #8

Source File: LocalRunner.java From storm-benchmark with Apache License 2.0

6 votes

private static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  LOG.info("running benchmark " + name);
  IBenchmark benchmark =  (IBenchmark) Runner.getApplicationFromName(PACKAGE + "." + name);
  Config config = new Config();
  config.putAll(Utils.readStormConfig());
  config.setDebug(true);
  StormTopology topology = benchmark.getTopology(config);
  LocalCluster localCluster = new LocalCluster();
  localCluster.submitTopology(name, config, topology);
  final int runtime = BenchmarkUtils.getInt(config, MetricsCollectorConfig.METRICS_TOTAL_TIME,
          MetricsCollectorConfig.DEFAULT_TOTAL_TIME);
  IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
  collector.run();
  try {
    Thread.sleep(runtime);
  } catch (InterruptedException e) {
    LOG.error("benchmark interrupted", e);
  }
  localCluster.shutdown();
}

Example #9

Source File: WordCountTopology.java From jstorm with Apache License 2.0

6 votes

/**
 * Main method
 */
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    if (args.length != 1) {
        throw new RuntimeException("Specify topology name");
    }

    int parallelism = 10;
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("word", new WordSpout(), parallelism);
    builder.setBolt("consumer", new ConsumerBolt(), parallelism)
            .fieldsGrouping("word", new Fields("word"));
    Config conf = new Config();
    conf.setNumStmgrs(parallelism);
/*
Set config here
*/

    StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
}

Example #10

Source File: Runner.java From storm-benchmark with Apache License 2.0

5 votes

public static void run(String name)
        throws ClassNotFoundException, IllegalAccessException,
        InstantiationException, AlreadyAliveException, InvalidTopologyException {
  if (name.startsWith("storm.benchmark.benchmarks")) {
    LOG.info("running benchmark " + name);
    runBenchmark((IBenchmark) getApplicationFromName(name));
  } else if (name.startsWith("storm.benchmark.tools.producer")) {
    LOG.info("running producer " + name);
    runProducer((IProducer) getApplicationFromName(name));
  } else {
    throw new RuntimeException(name + " is neither benchmark nor producer");
  }
}

Example #11

Source File: SequenceTopologyTool.java From jstorm with Apache License 2.0

5 votes

public void SetDPRCTopology() throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}

Example #12

Source File: SequenceTopology.java From jstorm with Apache License 2.0

5 votes

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        streamName = "SequenceTest";
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    SetBuilder(builder, conf);
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
}

Example #13

Source File: SequenceTopology.java From jstorm with Apache License 2.0

5 votes

public static void SetDPRCTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    // LinearDRPCTopologyBuilder builder = new LinearDRPCTopologyBuilder(
    // "exclamation");
    //
    // builder.addBolt(new TotalCount(), 3);
    //
    // Config conf = new Config();
    //
    // conf.setNumWorkers(3);
    // StormSubmitter.submitTopology("rpc", conf,
    // builder.createRemoteTopology());
    System.out.println("Please refer to com.alipay.dw.jstorm.example.drpc.ReachTopology");
}

Example #14

Source File: Runner.java From storm-benchmark with Apache License 2.0

5 votes

private static void runApplication(IApplication app)
        throws AlreadyAliveException, InvalidTopologyException {
  config.putAll(Utils.readStormConfig());
  String name = (String) config.get(Config.TOPOLOGY_NAME);
  topology = app.getTopology(config);
  StormSubmitter.submitTopology(name, config, topology);
}

Example #15

Source File: Runner.java From storm-benchmark with Apache License 2.0

5 votes

public static void runBenchmark(IBenchmark benchmark)
        throws AlreadyAliveException, InvalidTopologyException,
        ClassNotFoundException, IllegalAccessException, InstantiationException {
  runApplication(benchmark);
  if (isMetricsEnabled()) {
    IMetricsCollector collector = benchmark.getMetricsCollector(config, topology);
    collector.run();
  }
}

Example #16

Source File: TopologyNettyMgr.java From jstorm with Apache License 2.0

5 votes

public void rmTopology(String topologyId) {
    String topologyName;
    try {
        topologyName = Common.topologyIdToName(topologyId);
        setting.remove(topologyName);
        LOG.info("Remove {} netty metrics setting ", topologyName);
    } catch (InvalidTopologyException ignored) {
    }
}

Example #17

Source File: StormAbstractCloudLiveTest.java From brooklyn-library with Apache License 2.0

5 votes

private StormTopology createTopology()
        throws AlreadyAliveException, InvalidTopologyException {
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("word", new TestWordSpout(), 10);
    builder.setBolt("exclaim1", new ExclamationBolt(), 3).shuffleGrouping("word");
    builder.setBolt("exclaim2", new ExclamationBolt(), 2).shuffleGrouping("exclaim1");
    
    return builder.createTopology();
}

Example #18

Source File: ServiceHandler.java From jstorm with Apache License 2.0

5 votes

/**
 * @return Map[task id, component id]
 */
public Map<Integer, TaskInfo> mkTaskComponentAssignments(Map<Object, Object> conf, String topologyId)
        throws IOException, InvalidTopologyException, KeyNotFoundException {
    // we can directly pass stormConf from submit method ?
    Map<Object, Object> stormConf = StormConfig.read_nimbus_topology_conf(topologyId, data.getBlobStore());
    StormTopology rawTopology = StormConfig.read_nimbus_topology_code(topologyId, data.getBlobStore());
    StormTopology topology = Common.system_topology(stormConf, rawTopology);

    return Common.mkTaskInfo(stormConf, topology, topologyId);
}

Example #19

Source File: TMUdfStreamTopology.java From jstorm with Apache License 2.0

5 votes

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException {
    Map config = new Config();
    config.put(ConfigExtension.TOPOLOGY_MASTER_USER_DEFINED_STREAM_CLASS, "com.alipay.dw.jstorm.example.tm.TMUdfHandler");
    config.put(Config.TOPOLOGY_WORKERS, 2);

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("TMUdfSpout", new TMUdfSpout(), 2);
    builder.setBolt("TMUdfBolt", new TMUdfBolt(), 4);
    StormTopology topology = builder.createTopology();

    StormSubmitter.submitTopology("TMUdfTopology", config, topology);
}

Example #20

Source File: KafkaThroughput.java From flink-perf with Apache License 2.0

5 votes

public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, UnknownHostException, InterruptedException {
	final ParameterTool pt = ParameterTool.fromArgs(args);

	TopologyBuilder builder = new TopologyBuilder();
	BrokerHosts hosts = new ZkHosts(pt.getRequired("zookeeper"));
	SpoutConfig spoutConfig = new SpoutConfig(hosts, pt.getRequired("topic"), "/" + pt.getRequired("topic"), UUID.randomUUID().toString());
	spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
	KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
	builder.setSpout("source", kafkaSpout, pt.getInt("sourceParallelism"));

	builder.setBolt("sink", new Throughput.Sink(pt), pt.getInt("sinkParallelism")).noneGrouping("source");

	Config conf = new Config();
	conf.setDebug(false);

	if (!pt.has("local")) {
		conf.setNumWorkers(pt.getInt("par", 2));

		StormSubmitter.submitTopologyWithProgressBar("kafka-spout-"+pt.get("name", "no_name"), conf, builder.createTopology());
	} else {
		conf.setMaxTaskParallelism(pt.getInt("par", 2));

		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("kafka-spout", conf, builder.createTopology());

		Thread.sleep(300000);

		cluster.shutdown();
	}
}

Example #21

Source File: StormTopologySubmitter.java From samoa with Apache License 2.0

4 votes

public static void main(String[] args) throws IOException{
	Properties props = StormSamoaUtils.getProperties();
	
	String uploadedJarLocation = props.getProperty(StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
	if(uploadedJarLocation == null){
		logger.error("Invalid properties file. It must have key {}", 
				StormJarSubmitter.UPLOADED_JAR_LOCATION_KEY);
		return;
	}
	
	List<String> tmpArgs = new ArrayList<String>(Arrays.asList(args));
	int numWorkers = StormSamoaUtils.numWorkers(tmpArgs);
	
	args = tmpArgs.toArray(new String[0]);
	StormTopology stormTopo = StormSamoaUtils.argsToTopology(args);

	Config conf = new Config();
	conf.putAll(Utils.readStormConfig());
	conf.putAll(Utils.readCommandLineOpts());
	conf.setDebug(false);
	conf.setNumWorkers(numWorkers);
	
	String profilerOption = 
			props.getProperty(StormTopologySubmitter.YJP_OPTIONS_KEY);
	if(profilerOption != null){
		String topoWorkerChildOpts =  (String) conf.get(Config.TOPOLOGY_WORKER_CHILDOPTS);
		StringBuilder optionBuilder = new StringBuilder();
		if(topoWorkerChildOpts != null){
			optionBuilder.append(topoWorkerChildOpts);	
			optionBuilder.append(' ');
		}
		optionBuilder.append(profilerOption);
		conf.put(Config.TOPOLOGY_WORKER_CHILDOPTS, optionBuilder.toString());
	}

	Map<String, Object> myConfigMap = new HashMap<String, Object>(conf);
	StringWriter out = new StringWriter();

	try {
		JSONValue.writeJSONString(myConfigMap, out);
	} catch (IOException e) {
		System.out.println("Error in writing JSONString");
		e.printStackTrace();
		return;
	}
	
	Config config = new Config();
	config.putAll(Utils.readStormConfig());
	
	String nimbusHost = (String) config.get(Config.NIMBUS_HOST);
			
	NimbusClient nc = new NimbusClient(nimbusHost);
	String topologyName = stormTopo.getTopologyName();
	try {
		System.out.println("Submitting topology with name: " 
				+ topologyName);
		nc.getClient().submitTopology(topologyName, uploadedJarLocation,
				out.toString(), stormTopo.getStormBuilder().createTopology());
		System.out.println(topologyName + " is successfully submitted");

	} catch (AlreadyAliveException aae) {
		System.out.println("Fail to submit " + topologyName
				+ "\nError message: " + aae.get_msg());
	} catch (InvalidTopologyException ite) {
		System.out.println("Invalid topology for " + topologyName);
		ite.printStackTrace();
	} catch (TException te) {
		System.out.println("Texception for " + topologyName);
		te.printStackTrace();
	} 		
}

Example #22

Source File: Lancope.java From opensoc-streaming with Apache License 2.0

4 votes

public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
	
	TopologyRunner runner = new LancopeRunner();
	runner.initTopology(args, "lancope");
}

Example #23

Source File: SnapshotStateMaster.java From jstorm with Apache License 2.0

4 votes

public SnapshotStateMaster(TopologyContext context, OutputCollector outputCollector) {
    this.topologyId = context.getTopologyId();
    try {
        this.topologyName = Common.topologyIdToName(topologyId);
    } catch (InvalidTopologyException e) {
        LOG.error("Failed to convert topologyId to topologyName", e);
        throw new RuntimeException(e);
    }
    this.topology = context.getRawTopology();
    this.conf = context.getStormConf();
    this.outputCollector = outputCollector;
    this.context = context;

    String topologyStateOpClassName = ConfigExtension.getTopologyStateOperatorClass(conf);
    if (topologyStateOpClassName == null) {
        stateOperator = new DefaultTopologyStateOperator();
    } else {
        stateOperator = (ITopologyStateOperator) Utils.newInstance(topologyStateOpClassName);
    }
    stateOperator.init(context);

    Set<String> spoutIds = topology.get_spouts().keySet();
    Set<String> statefulBoltIds = TransactionCommon.getStatefulBolts(topology);
    Set<String> endBolts = TransactionCommon.getEndBolts(topology);
    Set<String> downstreamComponents = new HashSet<>(topology.get_bolts().keySet());

    spouts = componentToComponentTasks(context, spoutIds);
    statefulBolts = componentToComponentTasks(context, statefulBoltIds);
    downstreamComponents.removeAll(statefulBoltIds);
    nonStatefulBoltTasks = componentToComponentTasks(context, downstreamComponents);
    endBoltTasks = new HashSet<Integer>(context.getComponentsTasks(endBolts));
    snapshotState = new SnapshotState(context, spouts, statefulBolts, nonStatefulBoltTasks, endBoltTasks, stateOperator);

    SnapshotState commitState = ConfigExtension.resetTransactionTopologyState(conf) ? null : (SnapshotState) stateOperator.initState(topologyName);
    snapshotState.initState(commitState);

    LOG.info("topologySnapshotState: {}, isResetTopologyState: {}", snapshotState, ConfigExtension.resetTransactionTopologyState(conf));
    LOG.info("lastSuccessfulSnapshotState: {}", snapshotState.getLastSuccessfulBatch().statesInfo());

    this.batchSnapshotTimeout = ConfigExtension.getTransactionBatchSnapshotTimeout(conf);
    scheduledService = Executors.newSingleThreadScheduledExecutor();
    scheduledService.scheduleAtFixedRate(new Runnable() {
        @Override
        public void run() {
            expiredCheck();
        }
    }, batchSnapshotTimeout, batchSnapshotTimeout / 2, TimeUnit.SECONDS);

    this.lock = new ReentrantLock(true);
}

Example #24

Source File: DefaultTopologyValidator.java From jstorm with Apache License 2.0

4 votes

@Override
public void validate(String topologyName, Map topologyConf, StormTopology topology) throws InvalidTopologyException {
}

Example #25

Source File: ILocalCluster.java From jstorm with Apache License 2.0

4 votes

void submitTopology(String topologyName, Map conf, StormTopology topology)
throws AlreadyAliveException, InvalidTopologyException;

Example #26

Source File: ILocalCluster.java From jstorm with Apache License 2.0

4 votes

void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts)
throws AlreadyAliveException, InvalidTopologyException;

Example #27

Source File: Pcap.java From opensoc-streaming with Apache License 2.0

4 votes

public static void main(String[] args) throws ConfigurationException, Exception, InvalidTopologyException {
	
	TopologyRunner runner = new PcapRunner();
	runner.initTopology(args, "pcap");
}

Example #28

Source File: ILocalCluster.java From incubator-heron with Apache License 2.0

4 votes

@SuppressWarnings("rawtypes")
void submitTopology(String topologyName, Map conf, StormTopology topology) throws
    AlreadyAliveException, InvalidTopologyException;

Example #29

Source File: ConfigurableIngestTopology.java From cognition with Apache License 2.0

4 votes

void submitLocal(ILocalCluster cluster) throws AlreadyAliveException, InvalidTopologyException {
  stormConfig.setDebug(true);
  cluster.submitTopology(topologyName, stormConfig, builder.createTopology());
}

Example #30

Source File: ConfigurableIngestTopology.java From cognition with Apache License 2.0

4 votes

void submit() throws AlreadyAliveException, InvalidTopologyException {
  StormSubmitter.submitTopology(topologyName, stormConfig, builder.createTopology());
}