Java Code Examples for backtype.storm.utils.Utils

The following examples show how to use backtype.storm.utils.Utils. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: jstorm   Source File: ZookeeperManager.java    License: Apache License 2.0 6 votes vote down vote up
public static String getZKNodeData(String clusterName, String path) {
    String out = null;
    try {
        ClusterState clusterState = getAndCreateClusterState(clusterName);
        if (clusterState == null) {
            throw new IllegalStateException("Cluster state is null");
        }

        byte[] data = clusterState.get_data(PathUtils.normalize_path(path), false);
        if (data != null && data.length > 0) {
            Object obj = Utils.maybe_deserialize(data);
            if (obj != null){
                out = gson.toJson(obj);
            } else {
                out = new String(data);
            }
        }
    } catch (Exception e) {
        LOG.error("Get zookeeper data error!", e);
    }
    return out;
}
 
Example 2
Source Project: jstorm   Source File: rollback_topology.java    License: Apache License 2.0 6 votes vote down vote up
private static void rollbackTopology(String topologyName) {
    Map conf = Utils.readStormConfig();
    NimbusClient client = NimbusClient.getConfiguredClient(conf);
    try {
        // update jar
        client.getClient().rollbackTopology(topologyName);
        CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName);
    } catch (Exception e) {
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 3
Source Project: jstorm   Source File: RocksTTLDBCache.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Object get(String key) {
    for (Entry<Integer, ColumnFamilyHandle> entry : windowHandlers.entrySet()) {
        try {
            byte[] data = ttlDB.get(entry.getValue(), key.getBytes());
            if (data != null) {
                try {
                    return Utils.javaDeserialize(data);
                } catch (Exception e) {
                    LOG.error("Failed to deserialize obj of " + key);
                    ttlDB.remove(entry.getValue(), key.getBytes());
                    return null;
                }
            }
        } catch (Exception ignored) {
        }
    }

    return null;
}
 
Example 4
Source Project: jstorm   Source File: NettyUnitTest.java    License: Apache License 2.0 6 votes vote down vote up
private IConnection initNettyServer(int port) {
    ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();
    //ConcurrentHashMap<Integer, DisruptorQueue> deserializeCtrlQueues = new ConcurrentHashMap<Integer, DisruptorQueue>();

    WaitStrategy wait = (WaitStrategy)Utils.newInstance("com.lmax.disruptor.TimeoutBlockingWaitStrategy", 5, TimeUnit.MILLISECONDS);
    DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI,
            256, wait, false, 0, 0);
    Set<Integer> taskSet = new HashSet<Integer>();
    taskSet.add(1);
    IConnection server = context.bind(null, port, deserializeQueues, recvControlQueue, true, taskSet);

    WaitStrategy waitStrategy = new BlockingWaitStrategy();
    DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy, false, 0, 0);
    server.registerQueue(task, recvQueue);

    return server;
}
 
Example 5
Source Project: jstorm   Source File: list.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    NimbusClient client = null;
    try {
        Map conf = Utils.readStormConfig();
        client = NimbusClient.getConfiguredClient(conf);

        if (args.length > 0 && !StringUtils.isBlank(args[0])) {
            String topologyName = args[0];
            TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName);
            System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info));
        } else {
            ClusterSummary clusterSummary = client.getClient().getClusterInfo();
            System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary));
        }
    } catch (Exception e) {
        System.out.println(e.getMessage());
        e.printStackTrace();
        throw new RuntimeException(e);
    } finally {
        if (client != null) {
            client.close();
        }
    }
}
 
Example 6
Source Project: jstorm   Source File: StormConfig.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
public static Object readLocalObject(String topologyId, String readFile) throws IOException {
    String errMsg = "Failed to get topology configuration of " + topologyId + " file:" + readFile;

    byte[] bconf = FileUtils.readFileToByteArray(new File(readFile));
    if (bconf == null) {
        errMsg += ", failed to read";
        LOG.error(errMsg);
        throw new IOException(errMsg);
    }

    Object ret;
    try {
        ret = Utils.javaDeserialize(bconf);
    } catch (Exception e) {
        errMsg += ", failed to serialize the data";
        LOG.error(errMsg);
        throw new IOException(errMsg);
    }

    return ret;
}
 
Example 7
Source Project: storm-kafka-0.8-plus   Source File: KafkaBoltTest.java    License: Apache License 2.0 6 votes vote down vote up
private boolean verifyMessage(String key, String message) {
    long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1;
    ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer,
            new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset);
    MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next();
    Message kafkaMessage = messageAndOffset.message();
    ByteBuffer messageKeyBuffer = kafkaMessage.key();
    String keyString = null;
    String messageString = new String(Utils.toByteArray(kafkaMessage.payload()));
    if (messageKeyBuffer != null) {
        keyString = new String(Utils.toByteArray(messageKeyBuffer));
    }
    assertEquals(key, keyString);
    assertEquals(message, messageString);
    return true;
}
 
Example 8
Source Project: jstorm   Source File: ZooKeeperDataViewTest.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void init() {
    String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE;
    File file = new File(CONFIG_PATH);
    if (file.exists() == false) {
        SKIP = true;
        return;
    }

    try {
        zkobj = new Zookeeper();
        System.getProperties().setProperty("storm.conf.file", CONFIG_PATH);
        Map conf = Utils.readStormConfig();
        zk = zkobj.mkClient(conf,
                (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS),
                conf.get(Config.STORM_ZOOKEEPER_PORT),
                (String) conf.get(Config.STORM_ZOOKEEPER_ROOT));
        gson = new GsonBuilder().setPrettyPrinting().create();
    }catch(Throwable e) {
        e.printStackTrace();
        SKIP = true;
    }
}
 
Example 9
Source Project: jstorm   Source File: Heartbeat.java    License: Apache License 2.0 6 votes vote down vote up
private void initSupervisorInfo(Map conf) {
    Set<Integer> portList = JStormUtils.getDefaultSupervisorPortList(conf);
    if (!StormConfig.local_mode(conf)) {
        try {
            boolean isLocalIP = myHostName.equals("127.0.0.1") || myHostName.equals("localhost");
            if (isLocalIP) {
                throw new Exception("the hostname supervisor got is localhost");
            }
        } catch (Exception e1) {
            LOG.error("get supervisor host error!", e1);
            throw new RuntimeException(e1);
        }
        supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf);
    } else {
        supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf);
    }

    supervisorInfo.setVersion(Utils.getVersion());
    String buildTs = Utils.getBuildTime();
    supervisorInfo.setBuildTs(buildTs);
    LOG.info("jstorm version:{}, build ts:{}", supervisorInfo.getVersion(), supervisorInfo.getBuildTs());
}
 
Example 10
Source Project: storm-benchmark   Source File: Grep.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public StormTopology getTopology(Config config) {

  final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
  final int matBoltNum = BenchmarkUtils.getInt(config, FM_NUM, DEFAULT_MAT_BOLT_NUM);
  final int cntBoltNum = BenchmarkUtils.getInt(config, CM_NUM, DEFAULT_CNT_BOLT_NUM);
  final String ptnString = (String) Utils.get(config, PATTERN_STRING, DEFAULT_PATTERN_STR);

  spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme())));

  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(SPOUT_ID, spout, spoutNum);
  builder.setBolt(FM_ID, new FindMatchingSentence(ptnString), matBoltNum)
          .localOrShuffleGrouping(SPOUT_ID);
  builder.setBolt(CM_ID, new CountMatchingSentence(), cntBoltNum)
          .fieldsGrouping(FM_ID, new Fields(FindMatchingSentence.FIELDS));

  return builder.createTopology();
}
 
Example 11
Source Project: jstorm   Source File: AckTransactionBolt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple, ICollectorCallback callback) {
    if (anchors != null) {
        tuple.add(Utils.generateId(random));
    } else {
        tuple.add(0l);
    }
    delegate.emitDirect(taskId, streamId, null, tuple, callback);
}
 
Example 12
Source Project: jstorm   Source File: FeederBatchSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean isReady(long txid) {
    if(!_waitToEmit) return true;
    List allBatches = (List) RegisteredGlobalState.getState(_id);
    if(allBatches.size() > _masterEmitted) {
        _masterEmitted++;
        return true;
    } else {
        Utils.sleep(2);
        return false;
    }
}
 
Example 13
Source Project: jstorm   Source File: WorkerAssignment.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public String toJSONString() {
    Map<String, String> map = new HashMap<>();

    map.put(COMPONENTTONUM_TAG, Utils.to_json(componentToNum));
    map.put(MEM_TAG, String.valueOf(mem));
    map.put(CPU_TAG, String.valueOf(cpu));
    map.put(HOSTNAME_TAG, hostName);
    map.put(JVM_TAG, jvm);
    map.put(NODEID_TAG, getNodeId());
    map.put(PORT_TAG, String.valueOf(getPort()));

    return Utils.to_json(map);
}
 
Example 14
Source Project: incubator-heron   Source File: MultiStageAckingTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void nextTuple() {
  // We explicitly slow down the spout to avoid the stream mgr to be the bottleneck
  Utils.sleep(1);
  final String word = words[rand.nextInt(words.length)];
  // To enable acking, we need to emit tuple with MessageId, which is an object
  collector.emit(new Values(word), "MESSAGE_ID");
}
 
Example 15
Source Project: jstorm   Source File: TridentBoltExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void updateTaskCounts(List<Integer> tasks) {
    synchronized (_currBatch) {
        if (_currBatch != null) {
            Map<Integer, Integer> taskEmittedTuples = _currBatch.taskEmittedTuples;
            for (Integer task : tasks) {
                int newCount = Utils.get(taskEmittedTuples, task, 0) + 1;
                taskEmittedTuples.put(task, newCount);
            }
        }
    }
}
 
Example 16
Source Project: jstorm   Source File: FastWordCountTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1);
    isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false);
}
 
Example 17
Source Project: jstorm   Source File: JStormHelper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public <T> Object execute(T... args) {
    String topologyName = (String) args[0];

    try {
        checkError(conf, topologyName);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }

    Map stormConf = Utils.readStormConfig();
    stormConf.putAll(conf);
    Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null);
    for (Entry<String, Double> entry : ret.entrySet()) {
        String key = entry.getKey();
        Double value = entry.getValue();

        if (key.indexOf(MetricDef.FAILED_NUM) > 0) {
            Assert.assertTrue(key + " fail number should == 0", value == 0);
        } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }

        } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) {
            if (value == 0.0) {
                LOG.warn(key + ":" + value);
            } else {
                LOG.info(key + ":" + value);
            }
        }
    }

    return ret;
}
 
Example 18
Source Project: jstorm   Source File: WordSpout.java    License: Apache License 2.0 5 votes vote down vote up
public void nextTuple() {
    Utils.sleep(100);
    final String[] words = new String[] { "nathan", "mike", "jackson", "golda", "bertels" };
    final Random rand = new Random();
    final String word = words[rand.nextInt(words.length)];
    _collector.emit(new Values(word));
}
 
Example 19
Source Project: jstorm   Source File: TransactionCommon.java    License: Apache License 2.0 5 votes vote down vote up
public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) {
    Set<String> upstreamComponents = new HashSet<>();
    ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId);
    Set<GlobalStreamId> input = componentCommon.get_inputs().keySet();
    for (GlobalStreamId stream : input) {
        upstreamComponents.add(stream.get_componentId());
    }
    return upstreamComponents;
}
 
Example 20
Source Project: jstorm   Source File: RotatingTransactionalState.java    License: Apache License 2.0 5 votes vote down vote up
public void cleanupBefore(long txid) {
    SortedMap<Long, Object> toDelete = _curr.headMap(txid);
    for(long tx: new HashSet<>(toDelete.keySet())) {
        _curr.remove(tx);
        try {
            _state.delete(txPath(tx));
        } catch(RuntimeException e) {
            // Ignore NoNodeExists exceptions because when sync() it may populate _curr with stale data since
            // zookeeper reads are eventually consistent.
            if(!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) {
                throw e;
            }
        }
    }
}
 
Example 21
Source Project: jstorm   Source File: DefaultUncaughtExceptionHandler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void uncaughtException(Thread a, Throwable e) {
    try {
        Utils.handleUncaughtException(e);
    } catch (Error error) {
        LOG.info("Received error in main thread.. terminating server...", error);
        Runtime.getRuntime().exit(-2);
    }
}
 
Example 22
Source Project: jstorm   Source File: KeyRangeState.java    License: Apache License 2.0 5 votes vote down vote up
private void initKeyRangeState(int keyRange) {
    IRichCheckpointKvState<K, V, String> state =
            (IRichCheckpointKvState<K, V, String>) Utils.newInstance("com.alibaba.jstorm.hdfs.transaction.RocksDbHdfsState");
    state.setStateName(context.getThisComponentId() + "/" + String.valueOf(keyRange));
    state.init(context);
    keyRangeToState.put(keyRange, state);
}
 
Example 23
Source Project: storm-benchmark   Source File: PageViewCountTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void componentParallelismCouldBeSetThroughConfig() {
  StormBenchmark benchmark = new PageViewCount();
  Config config = new Config();
  config.put(PageViewCount.SPOUT_NUM, 3);
  config.put(PageViewCount.VIEW_NUM, 4);
  config.put(PageViewCount.COUNT_NUM, 5);

  StormTopology topology = benchmark.getTopology(config);
  assertThat(topology).isNotNull();
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.SPOUT_ID), 3);
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.VIEW_ID), 4);
  TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.COUNT_ID), 5);
}
 
Example 24
Source Project: jstorm   Source File: StormZkClusterState.java    License: Apache License 2.0 5 votes vote down vote up
public void setObject(String path, Object obj) throws Exception {
    if (obj instanceof byte[]) {
        cluster_state.set_data(path, (byte[]) obj);
    } else if (obj instanceof String) {
        cluster_state.set_data(path, ((String) obj).getBytes());
    } else {
        cluster_state.set_data(path, Utils.serialize(obj));
    }
}
 
Example 25
Source Project: jstorm   Source File: UIUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static Map<String, Object> getNimbusConf(String clusterName) {
    NimbusClient client = null;
    try {
        client = NimbusClientManager.getNimbusClient(clusterName);

        String jsonConf = client.getClient().getNimbusConf();
        Map<String, Object> nimbusConf =
                (Map<String, Object>) Utils.from_json(jsonConf);
        return nimbusConf;
    } catch (Exception e) {
        NimbusClientManager.removeClient(clusterName);
        LOG.error(e.getMessage(), e);
        return UIUtils.readUiConfig();
    }
}
 
Example 26
Source Project: eagle   Source File: UnitTopologyRunner.java    License: Apache License 2.0 5 votes vote down vote up
private void run(String topologyId,
                 int numOfTotalWorkers,
                 int numOfSpoutTasks,
                 int numOfRouterBolts,
                 int numOfAlertBolts,
                 int numOfPublishExecutors,
                 int numOfPublishTasks,
                 Config config,
                 boolean localMode) {

    backtype.storm.Config stormConfig = givenStormConfig == null ? new backtype.storm.Config() : givenStormConfig;
    // TODO: Configurable metric consumer instance number

    int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS) ? config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS;
    LOG.info("Set topology.message.timeout.secs as {}", messageTimeoutSecs);
    stormConfig.setMessageTimeoutSecs(messageTimeoutSecs);

    if (config.hasPath("metric")) {
        stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
    }

    stormConfig.setNumWorkers(numOfTotalWorkers);
    StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishExecutors, numOfPublishTasks, config).createTopology();

    if (localMode) {
        LOG.info("Submitting as local mode");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyId, stormConfig, topology);
        Utils.sleep(Long.MAX_VALUE);
    } else {
        LOG.info("Submitting as cluster mode");
        try {
            StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology);
        } catch (Exception ex) {
            LOG.error("fail submitting topology {}", topology, ex);
            throw new IllegalStateException(ex);
        }
    }
}
 
Example 27
Source Project: eagle   Source File: MockStreamReceiver.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This unit test is not to mock the end2end logic of correlation spout,
 * but simply generate some sample data for following bolts testing
 */
@Override
public void nextTuple() {
    PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1");
    LOG.info("Receive {}", event);
    collector.emit(outputStreamIds.get(
        // group by the first field in event i.e. name
        (int) (event.getPartitionKey() % outputStreamIds.size())),
        Collections.singletonList(event));
    Utils.sleep(500);
}
 
Example 28
Source Project: jstorm   Source File: AckTransactionSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
    if (messageId != null) {
        addPendingTuple(currBatchId, streamId, messageId, tuple);
        tuple.add(Utils.generateId(random));
    } else {
        // for non-anchor tuples, use 0 as default rootId
        tuple.add(0l);
    }
    return delegate.emit(streamId, tuple, null);
}
 
Example 29
Source Project: eagle   Source File: SampleClient2.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param args
 */
public static void main(String[] args) {
    AtomicLong base1 = new AtomicLong(System.currentTimeMillis());
    AtomicLong base2 = new AtomicLong(System.currentTimeMillis());
    AtomicLong count = new AtomicLong();

    Config config = ConfigFactory.load();

    try (KafkaProducer<String, String> proceduer = SampleClient1.createProceduer(config)) {
        while (true) {
            nextUuid = String.format(instanceUuidTemp, UUID.randomUUID().toString());
            nextReqId = String.format(reqIdTemp, UUID.randomUUID().toString());

            int hostIndex = 6;
            for (int i = 0; i < hostIndex; i++) {
                sendMetric(base1, base2, count, proceduer, i);
            }

            if (count.get() % 600 == 0) {
                System.out.println("send 600 LOG/FAILURE metric!");
            }

            Utils.sleep(3000);

        }
    }
}
 
Example 30
Source Project: jstorm   Source File: TridentBoltExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap<>(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector));
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));

    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap<>();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(),
                    COORD_STREAM(batchGroup),
                    new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}