Java Code Examples for backtype.storm.utils.Utils#readStormConfig()

The following examples show how to use backtype.storm.utils.Utils#readStormConfig() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TopologyMgmtResourceImpl.java    From eagle with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings( {"rawtypes", "unchecked"})
private Map getStormConf(List<StreamingCluster> clusters, String clusterId) throws Exception {
    Map<String, Object> stormConf = Utils.readStormConfig();
    if (clusterId == null) {
        stormConf.put(Config.NIMBUS_HOST, DEFAULT_NIMBUS_HOST);
        stormConf.put(Config.NIMBUS_THRIFT_PORT, DEFAULT_NIMBUS_THRIFT_PORT);
    } else {
        if (clusters == null) {
            clusters = dao.listClusters();
        }
        Optional<StreamingCluster> scOp = TopologyMgmtResourceHelper.findById(clusters, clusterId);
        StreamingCluster cluster;
        if (scOp.isPresent()) {
            cluster = scOp.get();
        } else {
            throw new Exception("Fail to find cluster: " + clusterId);
        }
        stormConf.put(Config.NIMBUS_HOST, cluster.getDeployments().getOrDefault(StreamingCluster.NIMBUS_HOST, DEFAULT_NIMBUS_HOST));
        stormConf.put(Config.NIMBUS_THRIFT_PORT, Integer.valueOf(cluster.getDeployments().get(StreamingCluster.NIMBUS_THRIFT_PORT)));
    }
    return stormConf;
}
 
Example 2
Source File: rollback_topology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 3
Source File: HealthCheck.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static HealthStatus check() {
    Map conf = Utils.readStormConfig();

    long timeout = ConfigExtension.getStormHealthTimeoutMs(conf);
    // 1. check panic dir
    String panicPath = ConfigExtension.getStormMachineResourcePanicCheckDir(conf);
    if (!isHealthyUnderPath(panicPath, timeout)) {
        return HealthStatus.PANIC;
    }

    // 2. check error dir
    String errorPath = ConfigExtension.getStormMachineResourceErrorCheckDir(conf);
    if (!isHealthyUnderPath(errorPath, timeout)) {
        return HealthStatus.ERROR;
    }

    // 3. check warn dir
    String warnPath = ConfigExtension.getStormMachineResourceWarningCheckDir(conf);
    if (!isHealthyUnderPath(warnPath, timeout)) {
        return HealthStatus.WARN;
    }

    return HealthStatus.INFO;
}
 
Example 4
Source File: activate.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 5
Source File: LocalUtils.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static Map getLocalConf(int port) {
    Map conf = Utils.readStormConfig();
    conf.putAll(getLocalBaseConf());

    List<String> zkServers = new ArrayList<>(1);
    zkServers.add("localhost");

    conf.put(Config.STORM_ZOOKEEPER_SERVERS, zkServers);
    conf.put(Config.STORM_ZOOKEEPER_PORT, port);

    ConfigExtension.setTopologyDebugRecvTuple(conf, true);
    conf.put(Config.TOPOLOGY_DEBUG, true);

    conf.put(ConfigExtension.TOPOLOGY_BACKPRESSURE_ENABLE, false);

    return conf;
}
 
Example 6
Source File: ZkTool.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    if (args.length < 2) {
        System.out.println("Invalid parameter");
        usage();
        return;
    }

    conf = Utils.readStormConfig();

    if (args[0].equalsIgnoreCase(READ_CMD)) {

        readData(args[1]);

    } else if (args[0].equalsIgnoreCase(RM_CMD)) {
        rmBakTopology(args[1]);
    } else if (args[0].equalsIgnoreCase(LIST_CMD)) {
        list(args[1]);
    } else if (args[0].equalsIgnoreCase(CLEAN_CMD)) {
        cleanTopology(args[1]);
    }

}
 
Example 7
Source File: deactivate.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 8
Source File: ZooKeeperDataViewTest.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void init() {
    String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE;
    File file = new File(CONFIG_PATH);
    if (file.exists() == false) {
        SKIP = true;
        return;
    }

    try {
        zkobj = new Zookeeper();
        System.getProperties().setProperty("storm.conf.file", CONFIG_PATH);
        Map conf = Utils.readStormConfig();
        zk = zkobj.mkClient(conf,
                (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS),
                conf.get(Config.STORM_ZOOKEEPER_PORT),
                (String) conf.get(Config.STORM_ZOOKEEPER_ROOT));
        gson = new GsonBuilder().setPrettyPrinting().create();
    }catch(Throwable e) {
        e.printStackTrace();
        SKIP = true;
    }
}
 
Example 9
Source File: complete_upgrade.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 10
Source File: TestReachTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
/**
 * @param args
 * @throws DRPCExecutionException
 * @throws TException
 */
public static void main(String[] args) throws Exception {
    
    if (args.length < 1) {
        throw new IllegalArgumentException("Invalid parameter");
    }
    Map conf = Utils.readStormConfig();
    // "foo.com/blog/1" "engineering.twitter.com/blog/5"
    DRPCClient client = new DRPCClient(conf, args[0], 4772);
    String result = client.execute(ReachTopology.TOPOLOGY_NAME, "tech.backtype.com/blog/123");
    
    System.out.println("\n!!! Drpc result:" + result);
}
 
Example 11
Source File: JStormHelper.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public <T> Object execute(T... args) {
    String topologyName = (String) args[0];

    try {
        checkError(conf, topologyName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    Map stormConf = Utils.readStormConfig();
    stormConf.putAll(conf);
    Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null);
    for (Entry<String, Double> entry : ret.entrySet()) {
        String key = entry.getKey();
        Double value = entry.getValue();

        if (key.indexOf(MetricDef.FAILED_NUM) > 0) {
            Assert.assertTrue(key + " fail number should == 0", value == 0);
        } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }

        } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }
        }
    }

    return ret;
}
 
Example 12
Source File: ProcessLauncher.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static int getSleepSeconds() {
    Map<Object, Object> conf;
    try {
        conf = Utils.readStormConfig();
    } catch (Exception e) {
        conf = new HashMap<>();
    }
    return ConfigExtension.getProcessLauncherSleepSeconds(conf);
}
 
Example 13
Source File: NimbusServer.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
    Thread.setDefaultUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler());
    // read configuration files
    @SuppressWarnings("rawtypes")
    Map config = Utils.readStormConfig();
    JStormServerUtils.startTaobaoJvmMonitor();
    NimbusServer instance = new NimbusServer();
    INimbus iNimbus = new DefaultInimbus();
    instance.launchServer(config, iNimbus);
}
 
Example 14
Source File: SandBoxMaker.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    Map<Object, Object> conf = Utils.readStormConfig();
    conf.put("java.sandbox.enable", true);
    SandBoxMaker maker = new SandBoxMaker(conf);
    try {
        System.out.println("sandboxPolicy:" + maker.sandboxPolicy("simple", new HashMap<String, String>()));
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 15
Source File: Supervisor.java    From jstorm with Apache License 2.0 5 votes vote down vote up
/**
 * start supervisor
 */
public void run() {
    SupervisorManger supervisorManager;
    try {
        Map<Object, Object> conf = Utils.readStormConfig();

        StormConfig.validate_distributed_mode(conf);

        createPid(conf);

        supervisorManager = mkSupervisor(conf, null);

        JStormUtils.redirectOutput("/dev/null");

        initShutdownHook(supervisorManager);

        while (!supervisorManager.isFinishShutdown()) {
            try {
                Thread.sleep(1000);
            } catch (InterruptedException ignored) {
            }
        }

    } catch (Throwable e) {
        if (e instanceof OutOfMemoryError) {
            LOG.error("Halting due to out of memory error...");
        }
        LOG.error("Fail to run supervisor ", e);
        System.exit(1);
    } finally {
        LOG.info("Shutdown supervisor!!!");
    }

}
 
Example 16
Source File: blacklist.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 17
Source File: BlobStoreTest.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private LocalFsBlobStore initLocalFs() {
    LocalFsBlobStore store = new LocalFsBlobStore();
    // Spy object that tries to mock the real object store
    LocalFsBlobStore spy = spy(store);
    Mockito.doNothing().when(spy).checkForBlobUpdate("test");
    Mockito.doNothing().when(spy).checkForBlobUpdate("other");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-WE");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-subject-DEF");
    Mockito.doNothing().when(spy).checkForBlobUpdate("test-empty-acls");
    Map conf = Utils.readStormConfig();
    conf.put(Config.STORM_LOCAL_DIR, baseFile.getAbsolutePath());
    conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
    spy.prepare(conf, null, null);
    return spy;
}
 
Example 18
Source File: config_value.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input key name");
    }

    String key = args[0];
    Map conf = Utils.readStormConfig();
    System.out.print("VALUE: " + String.valueOf(conf.get(key)));
}
 
Example 19
Source File: UserDefinedWorkerTopology.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public void verifyAssignment(String topologyName) {

            
            Set<ResourceWorkerSlot> spoutResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt1ResourceWorkerSlots = new HashSet<>();
            Set<ResourceWorkerSlot> bolt2ResourceWorkerSlots = new HashSet<>();
            
            NimbusClientWrapper client = new NimbusClientWrapper();
            try {
                Map nimbusConf = Utils.readStormConfig();
                client.init(nimbusConf);
                
                String topologyId = client.getClient().getTopologyId(topologyName);
                
                TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
                
                Assignment assignment = JStormHelper.getAssignment(topologyId, conf);
                Set<ResourceWorkerSlot> workerSet = assignment.getWorkers();
                
                List<ComponentSummary>  componentList = topologyInfo.get_components();
                for (ComponentSummary component : componentList) {
                    if (SPOUT_NAME.equals(component.get_name())) {
                        spoutResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT1_NAME.equals(component.get_name())) {
                        bolt1ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }else if (BOLT2_NAME.equals(component.get_name())) {
                        bolt2ResourceWorkerSlots = getComponentWorkers(component, workerSet);
                    }
                }
                
            }catch(Exception e) {
                Assert.fail("Fail to get workerSlots");
            }finally {
                client.cleanup();
            }
            
            
            verifySpoutAssignment(spoutResourceWorkerSlots);
            verifyBolt1Assignment(spoutResourceWorkerSlots, bolt1ResourceWorkerSlots);
            verifyBolt2Assignment(bolt1ResourceWorkerSlots, bolt2ResourceWorkerSlots);
            
        }
 
Example 20
Source File: StormConfig.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public static Map read_storm_config() {
    return Utils.readStormConfig();
}