org.apache.storm.generated.AuthorizationException Java Examples

The following examples show how to use org.apache.storm.generated.AuthorizationException. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlobStoreAPIWordCountTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    prepare();
    BlobStoreAPIWordCountTopology wc = new BlobStoreAPIWordCountTopology();
    try {
        File file = createFile(fileName);
        // Creating blob again before launching topology
        createBlobWithContent(key, store, file);

        // Blostore launch command with topology blobstore map
        // Here we are giving it a local name so that we can read from the file
        // bin/storm jar examples/storm-starter/storm-starter-topologies-0.11.0-SNAPSHOT.jar
        // org.apache.storm.starter.BlobStoreAPIWordCountTopology bl -c
        // topology.blobstore.map='{"key":{"localname":"blacklist.txt", "uncompress":"false"}}'
        wc.buildAndLaunchWordCountTopology(args);

        // Updating file few times every 5 seconds
        for (int i = 0; i < 10; i++) {
            updateBlobWithContent(key, store, updateFile(file));
            Utils.sleep(5000);
        }
    } catch (KeyAlreadyExistsException kae) {
        LOG.info("Key already exists {}", kae);
    } catch (AuthorizationException | KeyNotFoundException | IOException exp) {
        throw new RuntimeException(exp);
    }
}
 
Example #2
Source File: StormMain.java    From chuidiang-ejemplos with GNU Lesser General Public License v3.0 6 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException, AuthorizationException {
   TopologyBuilder builder = new TopologyBuilder();
   builder.setSpout(DATA_GENERATOR, new ASpout());
   builder.setBolt(DATA_CALCULATOR, new ABolt()).shuffleGrouping(DATA_GENERATOR);
   builder.setBolt(DATA_PRINTER, new DataPrinter()).shuffleGrouping(DATA_CALCULATOR).shuffleGrouping(DATA_GENERATOR);

   Config config = new Config();

   LocalCluster cluster = new LocalCluster();
   cluster.submitTopology(TOPOLOGY_NAME, config,
         builder.createTopology());

   Thread.sleep(100000);
   cluster.killTopology(TOPOLOGY_NAME);
   cluster.shutdown();
}
 
Example #3
Source File: WordCountApp.java    From java-study with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时,本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 
Example #4
Source File: WordCountApp.java    From java-study with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws InterruptedException, AlreadyAliveException, InvalidTopologyException {
 	//定义拓扑
     TopologyBuilder builder = new TopologyBuilder();
     builder.setSpout("word-reader" , new WordReader());
     builder.setBolt("word-normalizer" , new WordNormalizer()).shuffleGrouping("word-reader" );
     builder.setBolt("word-counter" , new WordCounter()).fieldsGrouping("word-normalizer" , new Fields("word"));
     StormTopology topology = builder.createTopology();
     //配置
     
     Config conf = new Config();
     String fileName ="words.txt" ;
     conf.put("fileName" , fileName );
     conf.setDebug(false);
 
      //运行拓扑
      System.out.println("开始...");
      if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
     	 System.out.println("远程模式");
          try {
	StormSubmitter.submitTopology(args[0], conf, topology);
} catch (AuthorizationException e) {
	e.printStackTrace();
}
    } else{//没有参数时,本地提交
      //启动本地模式
 	 System.out.println("本地模式");
      LocalCluster cluster = new LocalCluster();
      cluster.submitTopology("Getting-Started-Topologie" , conf , topology );
      Thread.sleep(5000);
      //关闭本地集群
      cluster.shutdown();
    }
      System.out.println("结束");
    
 }
 
Example #5
Source File: DemoApplication.java    From storm_spring_boot_demo with MIT License 5 votes vote down vote up
public static void main(String[] args)
        throws InvalidTopologyException, AuthorizationException, AlreadyAliveException, InterruptedException {
    ConfigurableApplicationContext context = SpringApplication.run(DemoApplication.class, args);
    AppMain appMain = context.getBean(AppMain.class);
    appMain.Laugher();
    SpringApplication.exit(context);
}
 
Example #6
Source File: PirkTopology.java    From incubator-retired-pirk with Apache License 2.0 5 votes vote down vote up
public static void runPirkTopology() throws PIRException
{
  // Set up Kafka parameters
  logger.info("Configuring Kafka.");
  String zkRoot = "/" + kafkaTopic + "_pirk_storm";
  BrokerHosts zkHosts = new ZkHosts(brokerZk);
  SpoutConfig kafkaConfig = new SpoutConfig(zkHosts, kafkaTopic, zkRoot, kafkaClientId);
  kafkaConfig.ignoreZkOffsets = forceFromStart;

  // Create conf
  logger.info("Retrieving Query and generating Storm conf.");
  Config conf = createStormConf();
  Query query = StormUtils.getQuery(useHdfs, hdfsUri, queryFile);
  conf.put(StormConstants.N_SQUARED_KEY, query.getNSquared().toString());
  conf.put(StormConstants.QUERY_INFO_KEY, query.getQueryInfo().toMap());

  // Configure this for different types of input data on Kafka.
  kafkaConfig.scheme = new SchemeAsMultiScheme(new PirkHashScheme(conf));

  // Create topology
  StormTopology topology = getPirkTopology(kafkaConfig);

  // Run topology
  logger.info("Submitting Pirk topology to Storm...");
  try
  {
    StormSubmitter.submitTopologyWithProgressBar(topologyName, conf, topology);
  } catch (AlreadyAliveException | InvalidTopologyException | AuthorizationException e)
  {
    throw new PIRException(e);
  }

}
 
Example #7
Source File: StormKafkaProcess.java    From BigData with GNU General Public License v3.0 5 votes vote down vote up
public static void main(String[] args)
		throws InterruptedException, InvalidTopologyException, AuthorizationException, AlreadyAliveException {

	String topologyName = "TSAS";// 元组名
	// Zookeeper主机地址,会自动选取其中一个
	ZkHosts zkHosts = new ZkHosts("192.168.230.128:2181,192.168.230.129:2181,192.168.230.131:2181");
	String topic = "trademx";
	String zkRoot = "/storm";// storm在Zookeeper上的根路径
	String id = "tsaPro";

	// 创建SpoutConfig对象
	SpoutConfig spontConfig = new SpoutConfig(zkHosts, topic, zkRoot, id);

	TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("kafka", new KafkaSpout(spontConfig), 2);
	builder.setBolt("AccBolt", new AccBolt()).shuffleGrouping("kafka");
	builder.setBolt("ToDbBolt", new ToDbBolt()).shuffleGrouping("AccBolt");

	Config config = new Config();
	config.setDebug(false);

	if (args.length == 0) { // 本地运行,用于测试
		LocalCluster localCluster = new LocalCluster();
		localCluster.submitTopology(topologyName, config, builder.createTopology());
		Thread.sleep(1000 * 3600);
		localCluster.killTopology(topologyName);
		localCluster.shutdown();
	} else { // 提交至集群运行
		StormSubmitter.submitTopology(topologyName, config, builder.createTopology());
	}

}
 
Example #8
Source File: BlobStoreAPIWordCountTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
private static void createBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
    throws AuthorizationException, KeyAlreadyExistsException, IOException, KeyNotFoundException {
    String stringBlobACL = "o::rwa";
    AccessControl blobACL = BlobStoreAclHandler.parseAccessControl(stringBlobACL);
    List<AccessControl> acls = new LinkedList<AccessControl>();
    acls.add(blobACL); // more ACLs can be added here
    SettableBlobMeta settableBlobMeta = new SettableBlobMeta(acls);
    AtomicOutputStream blobStream = clientBlobStore.createBlob(blobKey, settableBlobMeta);
    blobStream.write(readFile(file).toString().getBytes());
    blobStream.close();
}
 
Example #9
Source File: BlobStoreAPIWordCountTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public void buildAndLaunchWordCountTopology(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout("spout", new RandomSentenceSpout(), 5);
    builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
    builder.setBolt("filter", new FilterWords(), 6).shuffleGrouping("split");

    Config conf = new Config();
    conf.setDebug(true);
    try {
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(args[0], conf, builder.createTopology());
    } catch (InvalidTopologyException | AuthorizationException | AlreadyAliveException exp) {
        throw new RuntimeException(exp);
    }
}
 
Example #10
Source File: IPFraudDetectionTopology.java    From Building-Data-Streaming-Applications-with-Apache-Kafka with MIT License 4 votes vote down vote up
public static void main(String[] args) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
    Intialize(args[0]);
    logger.info("Successfully loaded Configuration ");


    BrokerHosts hosts = new ZkHosts(zkhost);
    SpoutConfig spoutConfig = new SpoutConfig(hosts, inputTopic, "/" + KafkaBroker, consumerGroup);
    spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
    spoutConfig.startOffsetTime = kafka.api.OffsetRequest.EarliestTime();
    KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);
    String[] partNames = {"status_code"};
    String[] colNames = {"date", "request_url", "protocol_type", "status_code"};

    DelimitedRecordHiveMapper mapper = new DelimitedRecordHiveMapper().withColumnFields(new Fields(colNames))
            .withPartitionFields(new Fields(partNames));


    HiveOptions hiveOptions;
    //make sure you change batch size and all paramtere according to requirement
    hiveOptions = new HiveOptions(metaStoreURI, dbName, tblName, mapper).withTxnsPerBatch(250).withBatchSize(2)
            .withIdleTimeout(10).withCallTimeout(10000000);

    logger.info("Creating Storm Topology");
    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout("KafkaSpout", kafkaSpout, 1);

    builder.setBolt("frauddetect", new FraudDetectorBolt()).shuffleGrouping("KafkaSpout");
    builder.setBolt("KafkaOutputBolt",
            new IPFraudKafkaBolt(zkhost, "kafka.serializer.StringEncoder", KafkaBroker, outputTopic), 1)
            .shuffleGrouping("frauddetect");

    builder.setBolt("HiveOutputBolt", new IPFraudHiveBolt(), 1).shuffleGrouping("frauddetect");
    builder.setBolt("HiveBolt", new HiveBolt(hiveOptions)).shuffleGrouping("HiveOutputBolt");

    Config conf = new Config();
    if (args != null && args.length > 1) {
        conf.setNumWorkers(3);
        logger.info("Submiting  topology to storm cluster");

        StormSubmitter.submitTopology(args[1], conf, builder.createTopology());
    } else {
        // Cap the maximum number of executors that can be spawned
        // for a component to 3
        conf.setMaxTaskParallelism(3);
        // LocalCluster is used to run locally
        LocalCluster cluster = new LocalCluster();
        logger.info("Submitting  topology to local cluster");
        cluster.submitTopology("KafkaLocal", conf, builder.createTopology());
        // sleep
        try {
            Thread.sleep(10000);
        } catch (InterruptedException e) {
            // TODO Auto-generated catch block
            logger.error("Exception ocuured" + e);
            cluster.killTopology("KafkaToplogy");
            logger.info("Shutting down cluster");
            cluster.shutdown();
        }
        cluster.shutdown();

    }

}
 
Example #11
Source File: AppMain.java    From storm_spring_boot_demo with MIT License 4 votes vote down vote up
public void Laugher() throws InvalidTopologyException, AuthorizationException, AlreadyAliveException, InterruptedException {
        Config config = new Config();
        remoteSubmit(stormProps,topologyBuilder,config);
//        localSubmit(stormProps.getTopologyName(),topologyBuilder,config);
    }
 
Example #12
Source File: AppMain.java    From storm_spring_boot_demo with MIT License 4 votes vote down vote up
private static void remoteSubmit(StormProps stormProps, TopologyBuilder builder, Config conf)
        throws InvalidTopologyException, AuthorizationException, AlreadyAliveException {
    conf.setNumWorkers(stormProps.getTopologyWorkers());
    conf.setMaxSpoutPending(stormProps.getTopologyMaxSpoutPending());
    StormSubmitter.submitTopology(stormProps.getTopologyName(), conf, builder.createTopology());
}
 
Example #13
Source File: WindowedQueryBolt_TestTopology.java    From streamline with Apache License 2.0 4 votes vote down vote up
private static Nimbus.Client runOnStormCluster(Config conf, StormTopology topology) throws AlreadyAliveException, InvalidTopologyException, AuthorizationException {
    Map<String, Object> clusterConf = Utils.readStormConfig();
    StormSubmitter.submitTopologyWithProgressBar(TOPO_NAME, conf, topology);
    return NimbusClient.getConfiguredClient(clusterConf).getClient();
}
 
Example #14
Source File: BlobStoreAPIWordCountTopology.java    From storm-net-adapter with Apache License 2.0 4 votes vote down vote up
private static void updateBlobWithContent(String blobKey, ClientBlobStore clientBlobStore, File file)
    throws KeyNotFoundException, AuthorizationException, IOException {
    AtomicOutputStream blobOutputStream = clientBlobStore.updateBlob(blobKey);
    blobOutputStream.write(readFile(file).toString().getBytes());
    blobOutputStream.close();
}