backtype.storm.utils.NimbusClient Java Examples

The following examples show how to use backtype.storm.utils.NimbusClient. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: complete_upgrade.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void completeTopology(String topologyName)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        client.getClient().completeUpgrade(topologyName);
        CommandLineUtil.success("Successfully submit command complete_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform complete_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #2
Source File: JStormHelper.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void cleanCluster() {
    try {
        NimbusClient client = getNimbusClient(null);

        ClusterSummary clusterSummary = client.getClient().getClusterInfo();

        List<TopologySummary> topologySummaries = clusterSummary.get_topologies();

        KillOptions killOption = new KillOptions();
        killOption.set_wait_secs(1);
        for (TopologySummary topologySummary : topologySummaries) {
            client.getClient().killTopologyWithOpts(topologySummary.get_name(), killOption);
            LOG.info("Successfully kill " + topologySummary.get_name());
        }
    } catch (Exception e) {
        if (client != null) {
            client.close();
            client = null;

        }

        LOG.error("Failed to kill all topology ", e);
    }
}
 
Example #3
Source File: list.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #4
Source File: ClusterAPIController.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@RequestMapping("/supervisor/summary")
public Map supervisors(@PathVariable String clusterName){
    Map ret;
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        ret = new HashMap<>();
        ret.put("supervisors", UIUtils.getSupervisorEntities(clusterSummary));
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 
Example #5
Source File: StormSubmitter.java    From eagle with Apache License 2.0 6 votes vote down vote up
private static boolean topologyNameExists(Map conf, String name) {
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        ClusterSummary summary = client.getClient().getClusterInfo();
        for (TopologySummary s : summary.get_topologies()) {
            if (s.get_name().equals(name)) {
                return true;
            }
        }
        return false;

    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        client.close();
    }
}
 
Example #6
Source File: StormExecutionRuntime.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void stop(Application<StormEnvironment, StormTopology> executor, com.typesafe.config.Config config) {
    String appId = config.getString("appId");
    LOG.info("Stopping topology {} ...", appId);
    if (Objects.equals(config.getString("mode"), ApplicationEntity.Mode.CLUSTER.name())) {
        Nimbus.Client stormClient = NimbusClient.getConfiguredClient(getStormConfig(config)).getClient();
        try {
            stormClient.killTopologyWithOpts(appId, this.killOptions);
        } catch (NotAliveException | TException e) {
            LOG.error("Failed to kill topology named {}, due to: {}",appId,e.getMessage(),e.getCause());
            throw new RuntimeException(e.getMessage(),e);
        }
    } else {
        getLocalCluster().killTopologyWithOpts(appId, this.killOptions);
    }
    LOG.info("Stopped topology {}", appId);
}
 
Example #7
Source File: rebalance.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void submitRebalance(String topologyName, RebalanceOptions options, Map conf) throws Exception {
    Map stormConf = Utils.readStormConfig();
    if (conf != null) {
        stormConf.putAll(conf);
    }

    NimbusClient client = null;
    try {
        client = NimbusClient.getConfiguredClient(stormConf);
        client.getClient().rebalance(topologyName, options);
    } catch (Exception e) {
        throw e;
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #8
Source File: rollback_topology.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #9
Source File: deactivate.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);
        client.getClient().deactivate(topologyName);
        System.out.println("Successfully submit command deactivate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #10
Source File: activate.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        client.getClient().activate(topologyName);

        System.out.println("Successfully submit command activate " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #11
Source File: UIUtils.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static boolean isValidSupervisorHost(String clusterName, String host) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<SupervisorEntity> supervisors = UIUtils.getSupervisorEntities(clusterSummary);
        for (SupervisorEntity s : supervisors) {
            if (s.getIp().equals(host) || s.getHost().equals(host)) {
                return true;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
    }
    return false;
}
 
Example #12
Source File: ClusterAPIController.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@RequestMapping("/nimbus/summary")
public Map nimbus(@PathVariable String clusterName){
    Map ret;
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        ret = new HashMap<>();
        ret.put("nimbus", UIUtils.getNimbusEntities(clusterSummary));
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 
Example #13
Source File: ClusterAPIController.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@RequestMapping("/topology/summary")
public Map topology(@PathVariable String clusterName){
    Map ret = new HashMap<>();
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        ClusterSummary clusterSummary = client.getClient().getClusterInfo();
        List<TopologyEntity> topologies = UIUtils.getTopologyEntities(clusterSummary);
        ret.put("topologies", topologies);
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 
Example #14
Source File: NimbusClientManager.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static NimbusClient getNimbusClient(String clusterName) throws Exception {
    Map conf = UIUtils.readUiConfig();
    NimbusClient client = clientManager.get(clusterName);
    if (client != null) {
        try {
            client.getClient().getVersion();
            LOG.info("get Nimbus Client from clientManager");
        } catch (Exception e) {
            LOG.info("Nimbus has been restarted, it begin to reconnect");
            client = null;
        }
    }

    if (client == null) {
        conf = UIUtils.resetZKConfig(conf, clusterName);
        client = NimbusClient.getConfiguredClient(conf);
        clientManager.put(clusterName, client);
    }

    return client;
}
 
Example #15
Source File: TopologyAPIController.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@RequestMapping("/metrics")
public Map summaryMetrics(@PathVariable String clusterName, @PathVariable String topology) {
    int window = 60;    //we only get the 60s window
    Map<String, Object> ret = new HashMap<>();
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);
        List<MetricInfo> infos = client.getClient().getMetrics(topology, MetaType.TOPOLOGY.getT());
        List<ChartSeries> metrics = UIUtils.getChartSeries(infos, window);
        ret.put("metrics", metrics);
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        ret = UIUtils.exceptionJson(e);
        LOG.error(e.getMessage(), e);
    }
    return ret;
}
 
Example #16
Source File: UIUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static Map<String, Object> getNimbusConf(String clusterName) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getNimbusConf();
        Map<String, Object> nimbusConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return nimbusConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return UIUtils.readUiConfig();
    }
}
 
Example #17
Source File: StormSubmitterForUCar.java    From PoseidonX with Apache License 2.0 5 votes vote down vote up
public static boolean topologyNameExists(NimbusClient client, Map conf, String name) {
    try {
        client.getClient().getTopologyInfoByName(name);
        return true;
    } catch (Exception e) {
        return false;
    }
}
 
Example #18
Source File: UIUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private static void flushClusterCache() {
    NimbusClient client = null;
    Map<String, ClusterEntity> clusterEntities = new HashMap<>();
    for (String name : UIUtils.clusterConfig.keySet()) {
        try {
            client = NimbusClientManager.getNimbusClient(name);
            clusterEntities.put(name, getClusterEntity(client.getClient().getClusterInfo(), name));
        } catch (Exception e) {
            NimbusClientManager.removeClient(name);
            LOG.error(e.getMessage(), e);
        }
    }
    clustersCache = clusterEntities;
}
 
Example #19
Source File: NimbusClientManager.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void removeClient(String clusterName) {
    if (StringUtils.isBlank(clusterName)) {
        clusterName = UIDef.DEFAULT_CLUSTER_NAME;
    }
    NimbusClient client = (NimbusClient) clientManager.remove(clusterName);
    if (client != null) {
        client.close();
    }
}
 
Example #20
Source File: StormSubmitter.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " +
                "StormSubmitter knows which jar to upload.");
    }

    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #21
Source File: UIUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static Map<String, Object> getTopologyConf(String clusterName, String topologyId) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getTopologyConf(topologyId);
        Map<String, Object> topologyConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return topologyConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return getNimbusConf(clusterName);
    }
}
 
Example #22
Source File: NimbusBlobStore.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map conf) {
    this.client = NimbusClient.getConfiguredClient(conf);
    if (conf != null) {
        this.bufferSize = JStormUtils.parseInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize);
    }
}
 
Example #23
Source File: NimbusBlobStore.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public boolean setClient(Map conf, NimbusClient client) {
    this.client = client;
    if (conf != null) {
        this.bufferSize = JStormUtils.parseInt(conf.get(Config.STORM_BLOBSTORE_INPUTSTREAM_BUFFER_SIZE_BYTES), bufferSize);
    }
    return true;
}
 
Example #24
Source File: gray_upgrade.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private static void upgradeTopology(String topologyName, String component, List<String> workers, int workerNum)
        throws Exception {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        String topologyId = client.getClient().getTopologyId(topologyName);
        Map stormConf = (Map) Utils.from_json(client.getClient().getTopologyConf(topologyId));
        // check if TM is a separate worker
        TopologyInfo topologyInfo = client.getClient().getTopologyInfo(topologyId);
        for (TaskSummary taskSummary : topologyInfo.get_tasks()) {
            if (!taskSummary.get_status().equalsIgnoreCase("active")) {
                CommandLineUtil.error("Some of the tasks are not in ACTIVE state, cannot perform the upgrade!");
                return;
            }
        }

        if (!ConfigExtension.isTmSingleWorker(stormConf, topologyInfo.get_topology().get_numWorkers())) {
            CommandLineUtil.error("Gray upgrade requires that topology master to be a single worker, " +
                    "cannot perform the upgrade!");
            return;
        }

        client.getClient().grayUpgrade(topologyName, component, workers, workerNum);
        CommandLineUtil.success("Successfully submit command gray_upgrade " + topologyName);
    } catch (Exception ex) {
        CommandLineUtil.error("Failed to perform gray_upgrade: " + ex.getMessage());
        ex.printStackTrace();
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #25
Source File: kill_topology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length == 1) {
            client.getClient().killTopology(topologyName);
        } else {
            int delaySeconds = Integer.parseInt(args[1]);
            KillOptions options = new KillOptions();
            options.set_wait_secs(delaySeconds);
            client.getClient().killTopologyWithOpts(topologyName, options);
        }

        System.out.println("Successfully submit command kill " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #26
Source File: blacklist.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length < 2) {
        throw new InvalidParameterException("Please input action and hostname");
    }

    String action = args[0];
    String hostname = args[1];

    NimbusClient client = null;
    try {

        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (action.equals("add"))
            client.getClient().setHostInBlackList(hostname);
        else {
            client.getClient().removeHostOutBlackList(hostname);
        }

        System.out.println("Successfully submit command blacklist with action:" + action + " and hostname :" + hostname);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #27
Source File: metrics_monitor.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length <= 1) {
        throw new InvalidParameterException("Please input topology name and enable flag");
    }

    String topologyName = args[0];
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        boolean isEnable = Boolean.valueOf(args[1]);

        MonitorOptions options = new MonitorOptions();
        options.set_isEnable(isEnable);

        client.getClient().metricMonitor(topologyName, options);

        String str = (isEnable) ? "enable" : "disable";
        System.out.println("Successfully submit command to " + str + " the monitor of " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #28
Source File: restart.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) {
    if (args == null || args.length == 0) {
        throw new InvalidParameterException("Please input topology name!");
    }

    String topologyName = args[0];

    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        System.out.println("It will take 15 ~ 100 seconds to restart, please wait patiently\n");

        if (args.length == 1) {
            client.getClient().restart(topologyName, null);
        } else {
            Map loadConf = Utils.loadConf(args[1]);
            String jsonConf = Utils.to_json(loadConf);
            System.out.println("New configuration:\n" + jsonConf);

            client.getClient().restart(topologyName, jsonConf);
        }

        System.out.println("Successfully submit command restart " + topologyName);
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example #29
Source File: StormSubmitter.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static boolean topologyNameExists(NimbusClient client, Map conf, String name) {
    try {
        client.getClient().getTopologyInfoByName(name);
        return true;
    } catch (Exception e) {
        return false;
    }
}
 
Example #30
Source File: StormSubmitter.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private static void submitJar(NimbusClient client, Map conf) {
    if (submittedJar == null) {
        try {
            LOG.info("Jar not uploaded to master yet. Submitting jar...");
            String localJar = System.getProperty("storm.jar");
            path = client.getClient().beginFileUpload();
            String[] pathCache = path.split("/");
            String uploadLocation = path + "/stormjar-" + pathCache[pathCache.length - 1] + ".jar";
            List<String> lib = (List<String>) conf.get(GenericOptionsParser.TOPOLOGY_LIB_NAME);
            Map<String, String> libPath = (Map<String, String>) conf.get(GenericOptionsParser.TOPOLOGY_LIB_PATH);
            if (lib != null && lib.size() != 0) {
                for (String libName : lib) {
                    String jarPath = path + "/lib/" + libName;
                    client.getClient().beginLibUpload(jarPath);
                    submitJar(conf, libPath.get(libName), jarPath, client);
                }

            } else {
                if (localJar == null) {
                    // no lib, no client jar
                    throw new RuntimeException("No client app jar found, please upload it");
                }
            }

            if (localJar != null) {
                submittedJar = submitJar(conf, localJar, uploadLocation, client);
            } else {
                // no client jar, but with lib jar
                client.getClient().finishFileUpload(uploadLocation);
            }
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    } else {
        LOG.info("Jar has already been uploaded to master. Will not submit again.");
    }
}