Java Code Examples for backtype.storm.utils.Utils

public static String getZKNodeData(String clusterName, String path) {
    String out = null;
    try {
        ClusterState clusterState = getAndCreateClusterState(clusterName);
        if (clusterState == null) {
            throw new IllegalStateException("Cluster state is null");
        }

        byte[] data = clusterState.get_data(PathUtils.normalize_path(path), false);
        if (data != null && data.length > 0) {
            Object obj = Utils.maybe_deserialize(data);
            if (obj != null){
                out = gson.toJson(obj);
            } else {
                out = new String(data);
            }
        }
    } catch (Exception e) {
        LOG.error("Get zookeeper data error!", e);
    }
    return out;
}
 

private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

@Override
public Object get(String key) {
    for (Entry<Integer, ColumnFamilyHandle> entry : windowHandlers.entrySet()) {
        try {
            byte[] data = ttlDB.get(entry.getValue(), key.getBytes());
            if (data != null) {
                try {
                    return Utils.javaDeserialize(data);
                } catch (Exception e) {
                    LOG.error("Failed to deserialize obj of " + key);
                    ttlDB.remove(entry.getValue(), key.getBytes());
                    return null;
                }
            }
        } catch (Exception ignored) {
        }
    }

    return null;
}
 

private IConnection initNettyServer(int port) {
    ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    //ConcurrentHashMap<Integer, DisruptorQueue> deserializeCtrlQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();

    WaitStrategy wait = (WaitStrategy)Utils.newInstance("com.lmax.disruptor.TimeoutBlockingWaitStrategy", 5, TimeUnit.MILLISECONDS);
    DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI,
            256, wait, false, 0, 0);
    Set<Integer> taskSet = new HashSet<Integer>();
    taskSet.add(1);
    IConnection server = context.bind(null, port, deserializeQueues, recvControlQueue, true, taskSet);

    WaitStrategy waitStrategy = new BlockingWaitStrategy();
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy, false, 0, 0);
    server.registerQueue(task, recvQueue);

    return server;
}
 

public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 

@SuppressWarnings("unchecked")
public static Object readLocalObject(String topologyId, String readFile) throws IOException {
    String errMsg = "Failed to get topology configuration of " + topologyId + " file:" + readFile;

    byte[] bconf = FileUtils.readFileToByteArray(new File(readFile));
    if (bconf == null) {
        errMsg += ", failed to read";
        LOG.error(errMsg);
        throw new IOException(errMsg);
    }

    Object ret;
    try {
        ret = Utils.javaDeserialize(bconf);
    } catch (Exception e) {
        errMsg += ", failed to serialize the data";
        LOG.error(errMsg);
        throw new IOException(errMsg);
    }

    return ret;
}
 

private boolean verifyMessage(String key, String message) {
    long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
    ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
            new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
    MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
    Message kafkaMessage = messageAndOffset.message();
    ByteBuffer messageKeyBuffer = kafkaMessage.key();
    String keyString = null;
    String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
    if (messageKeyBuffer != null) {
        keyString = new String(Utils.toByteArray(messageKeyBuffer));
    }
    assertEquals(key, keyString);
    assertEquals(message, messageString);
    return true;
}
 

@BeforeClass
public static void init() {
    String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE;
    File file = new File(CONFIG_PATH);
    if (file.exists() == false) {
        SKIP = true;
        return;
    }

    try {
        zkobj = new Zookeeper();
        System.getProperties().setProperty("storm.conf.file", CONFIG_PATH);
        Map conf = Utils.readStormConfig();
        zk = zkobj.mkClient(conf,
                (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS),
                conf.get(Config.STORM_ZOOKEEPER_PORT),
                (String) conf.get(Config.STORM_ZOOKEEPER_ROOT));
        gson = new GsonBuilder().setPrettyPrinting().create();
    }catch(Throwable e) {
        e.printStackTrace();
        SKIP = true;
    }
}
 

private void initSupervisorInfo(Map conf) {
    Set<Integer> portList = JStormUtils.getDefaultSupervisorPortList(conf);
    if (!StormConfig.local_mode(conf)) {
        try {
            boolean isLocalIP = myHostName.equals("127.0.0.1") || myHostName.equals("localhost");
            if (isLocalIP) {
                throw new Exception("the hostname supervisor got is localhost");
            }
        } catch (Exception e1) {
            LOG.error("get supervisor host error!", e1);
            throw new RuntimeException(e1);
        }
        supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf);
    } else {
        supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf);
    }

    supervisorInfo.setVersion(Utils.getVersion());
    String buildTs = Utils.getBuildTime();
    supervisorInfo.setBuildTs(buildTs);
    LOG.info("jstorm version:{}, build ts:{}", supervisorInfo.getVersion(), supervisorInfo.getBuildTs());
}
 

@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int matBoltNum = BenchmarkUtils.getInt(config, FM_NUM, DEFAULT_MAT_BOLT_NUM);
  final int cntBoltNum = BenchmarkUtils.getInt(config, CM_NUM, DEFAULT_CNT_BOLT_NUM);
  final String ptnString = (String) Utils.get(config, PATTERN_STRING, DEFAULT_PATTERN_STR);

  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(FM_ID, new FindMatchingSentence(ptnString), matBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(CM_ID, new CountMatchingSentence(), cntBoltNum)
          .fieldsGrouping(FM_ID, new Fields(FindMatchingSentence.FIELDS));

  return builder.createTopology();
}
 

@Override
public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple, ICollectorCallback callback) {
    if (anchors != null) {
        tuple.add(Utils.generateId(random));
    } else {
        tuple.add(0l);
    }
    delegate.emitDirect(taskId, streamId, null, tuple, callback);
}
 

@Override
public boolean isReady(long txid) {
    if(!_waitToEmit) return true;
    List allBatches = (List) RegisteredGlobalState.getState(_id);
    if(allBatches.size() > _masterEmitted) {
        _masterEmitted++;
        return true;
    } else {
        Utils.sleep(2);
        return false;
    }
}
 

@Override
public String toJSONString() {
    Map<String, String> map = new HashMap<>();

    map.put(COMPONENTTONUM_TAG, Utils.to_json(componentToNum));
    map.put(MEM_TAG, String.valueOf(mem));
    map.put(CPU_TAG, String.valueOf(cpu));
    map.put(HOSTNAME_TAG, hostName);
    map.put(JVM_TAG, jvm);
    map.put(NODEID_TAG, getNodeId());
    map.put(PORT_TAG, String.valueOf(getPort()));

    return Utils.to_json(map);
}
 

public void nextTuple() {
  // We explicitly slow down the spout to avoid the stream mgr to be the bottleneck
  Utils.sleep(1);
  final String word = words[rand.nextInt(words.length)];
  // To enable acking, we need to emit tuple with MessageId, which is an object
  collector.emit(new Values(word), "MESSAGE_ID");
}
 

private void updateTaskCounts(List<Integer> tasks) {
    synchronized (_currBatch) {
        if (_currBatch != null) {
            Map<Integer, Integer> taskEmittedTuples = _currBatch.taskEmittedTuples;
            for (Integer task : tasks) {
                int newCount = Utils.get(taskEmittedTuples, task, 0) + 1;
                taskEmittedTuples.put(task, newCount);
            }
        }
    }
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1);
    isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false);
}
 

@Override
public <T> Object execute(T... args) {
    String topologyName = (String) args[0];

    try {
        checkError(conf, topologyName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    Map stormConf = Utils.readStormConfig();
    stormConf.putAll(conf);
    Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null);
    for (Entry<String, Double> entry : ret.entrySet()) {
        String key = entry.getKey();
        Double value = entry.getValue();

        if (key.indexOf(MetricDef.FAILED_NUM) > 0) {
            Assert.assertTrue(key + " fail number should == 0", value == 0);
        } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }

        } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }
        }
    }

    return ret;
}
 

public void nextTuple() {
    Utils.sleep(100);
    final String[] words = new String[] { "nathan", "mike", "jackson", "golda", "bertels" };
    final Random rand = new Random();
    final String word = words[rand.nextInt(words.length)];
    _collector.emit(new Values(word));
}
 

public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) {
    Set<String> upstreamComponents = new HashSet<>();
    ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId);
    Set<GlobalStreamId> input = componentCommon.get_inputs().keySet();
    for (GlobalStreamId stream : input) {
        upstreamComponents.add(stream.get_componentId());
    }
    return upstreamComponents;
}
 

public void cleanupBefore(long txid) {
    SortedMap<Long, Object> toDelete = _curr.headMap(txid);
    for(long tx: new HashSet<>(toDelete.keySet())) {
        _curr.remove(tx);
        try {
            _state.delete(txPath(tx));
        } catch(RuntimeException e) {
            // Ignore NoNodeExists exceptions because when sync() it may populate _curr with stale data since
            // zookeeper reads are eventually consistent.
            if(!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
                throw e;
            }
        }
    }
}
 

@Override
public void uncaughtException(Thread a, Throwable e) {
    try {
        Utils.handleUncaughtException(e);
    } catch (Error error) {
        LOG.info("Received error in main thread.. terminating server...", error);
        Runtime.getRuntime().exit(-2);
    }
}
 

private void initKeyRangeState(int keyRange) {
    IRichCheckpointKvState<K, V, String> state =
            (IRichCheckpointKvState<K, V, String>) Utils.newInstance("com.alibaba.jstorm.hdfs.transaction.RocksDbHdfsState");
    state.setStateName(context.getThisComponentId() + "/" + String.valueOf(keyRange));
    state.init(context);
    keyRangeToState.put(keyRange, state);
}
 

@Test
public void componentParallelismCouldBeSetThroughConfig() {
  StormBenchmark benchmark = new PageViewCount();
  Config config = new Config();
  config.put(PageViewCount.SPOUT_NUM, 3);
  config.put(PageViewCount.VIEW_NUM, 4);
  config.put(PageViewCount.COUNT_NUM, 5);

  StormTopology topology = benchmark.getTopology(config);
  assertThat(topology).isNotNull();
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.SPOUT_ID), 3);
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.VIEW_ID), 4);
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.COUNT_ID), 5);
}
 

public void setObject(String path, Object obj) throws Exception {
    if (obj instanceof byte[]) {
        cluster_state.set_data(path, (byte[]) obj);
    } else if (obj instanceof String) {
        cluster_state.set_data(path, ((String) obj).getBytes());
    } else {
        cluster_state.set_data(path, Utils.serialize(obj));
    }
}
 

public static Map<String, Object> getNimbusConf(String clusterName) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getNimbusConf();
        Map<String, Object> nimbusConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return nimbusConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return UIUtils.readUiConfig();
    }
}
 

private void run(String topologyId,
                 int numOfTotalWorkers,
                 int numOfSpoutTasks,
                 int numOfRouterBolts,
                 int numOfAlertBolts,
                 int numOfPublishExecutors,
                 int numOfPublishTasks,
                 Config config,
                 boolean localMode) {

    backtype.storm.Config stormConfig = givenStormConfig == null ? new backtype.storm.Config() : givenStormConfig;
    // TODO: Configurable metric consumer instance number

    int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS) ? config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS;
    LOG.info("Set topology.message.timeout.secs as {}", messageTimeoutSecs);
    stormConfig.setMessageTimeoutSecs(messageTimeoutSecs);

    if (config.hasPath("metric")) {
        stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
    }

    stormConfig.setNumWorkers(numOfTotalWorkers);
    StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishExecutors, numOfPublishTasks, config).createTopology();

    if (localMode) {
        LOG.info("Submitting as local mode");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyId, stormConfig, topology);
        Utils.sleep(Long.MAX_VALUE);
    } else {
        LOG.info("Submitting as cluster mode");
        try {
            StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology);
        } catch (Exception ex) {
            LOG.error("fail submitting topology {}", topology, ex);
            throw new IllegalStateException(ex);
        }
    }
}
 

/**
 * This unit test is not to mock the end2end logic of correlation spout,
 * but simply generate some sample data for following bolts testing
 */
@Override
public void nextTuple() {
    PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1");
    LOG.info("Receive {}", event);
    collector.emit(outputStreamIds.get(
        // group by the first field in event i.e. name
        (int) (event.getPartitionKey() % outputStreamIds.size())),
        Collections.singletonList(event));
    Utils.sleep(500);
}
 

@Override
public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
    if (messageId != null) {
        addPendingTuple(currBatchId, streamId, messageId, tuple);
        tuple.add(Utils.generateId(random));
    } else {
        // for non-anchor tuples, use 0 as default rootId
        tuple.add(0l);
    }
    return delegate.emit(streamId, tuple, null);
}
 

/**
 * @param args
 */
public static void main(String[] args) {
    AtomicLong base1 = new AtomicLong(System.currentTimeMillis());
    AtomicLong base2 = new AtomicLong(System.currentTimeMillis());
    AtomicLong count = new AtomicLong();

    Config config = ConfigFactory.load();

    try (KafkaProducer<String, String> proceduer = SampleClient1.createProceduer(config)) {
        while (true) {
            nextUuid = String.format(instanceUuidTemp, UUID.randomUUID().toString());
            nextReqId = String.format(reqIdTemp, UUID.randomUUID().toString());

            int hostIndex = 6;
            for (int i = 0; i < hostIndex; i++) {
                sendMetric(base1, base2, count, proceduer, i);
            }

            if (count.get() % 600 == 0) {
                System.out.println("send 600 LOG/FAILURE metric!");
            }

            Utils.sleep(3000);

        }
    }
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap<>(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector));
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));

    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap<>();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(),
                    COORD_STREAM(batchGroup),
                    new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}