Java Code Examples for backtype.storm.utils.Utils#readStormConfig()

The following examples show how to use backtype.storm.utils.Utils#readStormConfig() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: eagle   File: TopologyMgmtResourceImpl.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings( {"rawtypes", "unchecked"})
private Map getStormConf(List<StreamingCluster> clusters, String clusterId) throws Exception {
    Map<String, Object> stormConf = Utils.readStormConfig();
    if (clusterId == null) {
        stormConf.put(Config.NIMBUS_HOST, DEFAULT_NIMBUS_HOST);
        stormConf.put(Config.NIMBUS_THRIFT_PORT, DEFAULT_NIMBUS_THRIFT_PORT);
    } else {
        if (clusters == null) {
            clusters = dao.listClusters();
        }
        Optional<StreamingCluster> scOp = TopologyMgmtResourceHelper.findById(clusters, clusterId);
        StreamingCluster cluster;
        if (scOp.isPresent()) {
            cluster = scOp.get();
        } else {
            throw new Exception("Fail to find cluster: " + clusterId);
        }
        stormConf.put(Config.NIMBUS_HOST, cluster.getDeployments().getOrDefault(StreamingCluster.NIMBUS_HOST, DEFAULT_NIMBUS_HOST));
        stormConf.put(Config.NIMBUS_THRIFT_PORT, Integer.valueOf(cluster.getDeployments().get(StreamingCluster.NIMBUS_THRIFT_PORT)));
    }
    return stormConf;
}
 
Example 2
Source Project: jstorm   File: complete_upgrade.java    License: Apache License 2.0 6 votes vote down vote up
private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 3
Source Project: jstorm   File: ZooKeeperDataViewTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void init() {
    String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE;
    File file = new File(CONFIG_PATH);
    if (file.exists() == false) {
        SKIP = true;
        return;
    }

    try {
        zkobj = new Zookeeper();
        System.getProperties().setProperty("storm.conf.file", CONFIG_PATH);
        Map conf = Utils.readStormConfig();
        zk = zkobj.mkClient(conf,
                (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS),
                conf.get(Config.STORM_ZOOKEEPER_PORT),
                (String) conf.get(Config.STORM_ZOOKEEPER_ROOT));
        gson = new GsonBuilder().setPrettyPrinting().create();
    }catch(Throwable e) {
        e.printStackTrace();
        SKIP = true;
    }
}
 
Example 4
Source Project: jstorm   File: deactivate.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 5
Source Project: jstorm   File: ZkTool.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    if (args.length < 2) {
        System.out.println("Invalid parameter");
        usage();
        return;
    }

    conf = Utils.readStormConfig();

    if (args[0].equalsIgnoreCase(READ_CMD)) {

        readData(args[1]);

    } else if (args[0].equalsIgnoreCase(RM_CMD)) {
        rmBakTopology(args[1]);
    } else if (args[0].equalsIgnoreCase(LIST_CMD)) {
        list(args[1]);
    } else if (args[0].equalsIgnoreCase(CLEAN_CMD)) {
        cleanTopology(args[1]);
    }

}
 
Example 6
Source Project: jstorm   File: LocalUtils.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static Map getLocalConf(int port) {
    Map conf = Utils.readStormConfig();
    conf.putAll(getLocalBaseConf());

    List<String> zkServers = new ArrayList<>(1);
    zkServers.add("localhost");

    conf.put(Config.STORM_ZOOKEEPER_SERVERS, zkServers);
    conf.put(Config.STORM_ZOOKEEPER_PORT, port);

    ConfigExtension.setTopologyDebugRecvTuple(conf, true);
    conf.put(Config.TOPOLOGY_DEBUG, true);

    conf.put(ConfigExtension.TOPOLOGY_BACKPRESSURE_ENABLE, false);

    return conf;
}
 
Example 7
Source Project: jstorm   File: activate.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 8
Source Project: jstorm   File: HealthCheck.java    License: Apache License 2.0 6 votes vote down vote up
public static HealthStatus check() {
    Map conf = Utils.readStormConfig();

    long timeout = ConfigExtension.getStormHealthTimeoutMs(conf);
    // 1. check panic dir
    String panicPath = ConfigExtension.getStormMachineResourcePanicCheckDir(conf);
    if (!isHealthyUnderPath(panicPath, timeout)) {
        return HealthStatus.PANIC;
    }

    // 2. check error dir
    String errorPath = ConfigExtension.getStormMachineResourceErrorCheckDir(conf);
    if (!isHealthyUnderPath(errorPath, timeout)) {
        return HealthStatus.ERROR;
    }

    // 3. check warn dir
    String warnPath = ConfigExtension.getStormMachineResourceWarningCheckDir(conf);
    if (!isHealthyUnderPath(warnPath, timeout)) {
        return HealthStatus.WARN;
    }

    return HealthStatus.INFO;
}
 
Example 9
Source Project: jstorm   File: rollback_topology.java    License: Apache License 2.0 6 votes vote down vote up
private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 10
Source Project: jstorm   File: config_value.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input key name");
    }

    String key = args[0];
    Map conf = Utils.readStormConfig();
    System.out.print("VALUE: " + String.valueOf(conf.get(key)));
}
 
Example 11
Source Project: jstorm   File: JStormHelper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <T> Object execute(T... args) {
    String topologyName = (String) args[0];

    try {
        checkError(conf, topologyName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    Map stormConf = Utils.readStormConfig();
    stormConf.putAll(conf);
    Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null);
    for (Entry<String, Double> entry : ret.entrySet()) {
        String key = entry.getKey();
        Double value = entry.getValue();

        if (key.indexOf(MetricDef.FAILED_NUM) > 0) {
            Assert.assertTrue(key + " fail number should == 0", value == 0);
        } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }

        } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }
        }
    }

    return ret;
}
 
Example 12
Source Project: jstorm   File: TestReachTopology.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param args
 * @throws DRPCExecutionException
 * @throws TException
 */
public static void main(String[] args) throws Exception {
    
    if (args.length < 1) {
        throw new IllegalArgumentException("Invalid parameter");
    }
    Map conf = Utils.readStormConfig();
    // "foo.com/blog/1" "engineering.twitter.com/blog/5"
    DRPCClient client = new DRPCClient(conf, args[0], 4772);
    String result = client.execute(ReachTopology.TOPOLOGY_NAME, "tech.backtype.com/blog/123");
    
    System.out.println("\n!!! Drpc result:" + result);
}
 
Example 13
Source Project: jstorm   File: ProcessLauncher.java    License: Apache License 2.0 5 votes vote down vote up
public static int getSleepSeconds() {
    Map<Object, Object> conf;
    try {
        conf = Utils.readStormConfig();
    } catch (Exception e) {
        conf = new HashMap<>();
    }
    return ConfigExtension.getProcessLauncherSleepSeconds(conf);
}
 
Example 14
Source Project: jstorm   File: NimbusServer.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler());
    // read configuration files
    @SuppressWarnings("rawtypes")
    Map config = Utils.readStormConfig();
    JStormServerUtils.startTaobaoJvmMonitor();
    NimbusServer instance = new NimbusServer();
    INimbus iNimbus = new DefaultInimbus();
    instance.launchServer(config, iNimbus);
}
 
Example 15
Source Project: jstorm   File: SandBoxMaker.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    Map<Object, Object> conf = Utils.readStormConfig();
    conf.put("java.sandbox.enable", true);
    SandBoxMaker maker = new SandBoxMaker(conf);
    try {
        System.out.println("sandboxPolicy:" + maker.sandboxPolicy("simple", new HashMap<String, String>()));
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 16
Source Project: jstorm   File: Supervisor.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * start supervisor
 */
public void run() {
    SupervisorManger supervisorManager;
    try {
        Map<Object, Object> conf = Utils.readStormConfig();

        StormConfig.validate_distributed_mode(conf);

        createPid(conf);

        supervisorManager = mkSupervisor(conf, null);

        JStormUtils.redirectOutput("/dev/null");

        initShutdownHook(supervisorManager);

        while (!supervisorManager.isFinishShutdown()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ignored) {
            }
        }

    } catch (Throwable e) {
        if (e instanceof OutOfMemoryError) {
            LOG.error("Halting due to out of memory error...");
        }
        LOG.error("Fail to run supervisor ", e);
        System.exit(1);
    } finally {
        LOG.info("Shutdown supervisor!!!");
    }

}
 
Example 17
Source Project: jstorm   File: blacklist.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 18
Source Project: jstorm   File: BlobStoreTest.java    License: Apache License 2.0 5 votes vote down vote up
private LocalFsBlobStore initLocalFs() {
    LocalFsBlobStore store = new LocalFsBlobStore();
    // Spy object that tries to mock the real object store
    LocalFsBlobStore spy = spy(store);
    Mockito.doNothing().when(spy).checkForBlobUpdate("test");
    Mockito.doNothing().when(spy).checkForBlobUpdate("other");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls");
    Map conf = Utils.readStormConfig();
    conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
    spy.prepare(conf, null, null);
    return spy;
}
 
Example 19
public void verifyAssignment(String topologyName) {

            
            Set<ResourceWorkerSlot> spoutResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt1ResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt2ResourceWorkerSlots = new HashSet<>();
            
            NimbusClientWrapper client = new NimbusClientWrapper();
            try {
                Map nimbusConf = Utils.readStormConfig();
                client.init(nimbusConf);
                
                String topologyId = client.getClient().getTopologyId(topologyName);
                
                TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
                
                Assignment assignment = JStormHelper.getAssignment(topologyId, conf);
                Set<ResourceWorkerSlot> workerSet = assignment.getWorkers();
                
                List<ComponentSummary>  componentList = topologyInfo.get_components();
                for (ComponentSummary component : componentList) {
                    if (SPOUT_NAME.equals(component.get_name())) {
                        spoutResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT1_NAME.equals(component.get_name())) {
                        bolt1ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT2_NAME.equals(component.get_name())) {
                        bolt2ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }
                }
                
            }catch(Exception e) {
                Assert.fail("Fail to get workerSlots");
            }finally {
                client.cleanup();
            }
            
            
            verifySpoutAssignment(spoutResourceWorkerSlots);
            verifyBolt1Assignment(spoutResourceWorkerSlots, bolt1ResourceWorkerSlots);
            verifyBolt2Assignment(bolt1ResourceWorkerSlots, bolt2ResourceWorkerSlots);
            
        }
 
Example 20
Source Project: jstorm   File: StormConfig.java    License: Apache License 2.0 4 votes vote down vote up
public static Map read_storm_config() {
    return Utils.readStormConfig();
}