Java Code Examples for backtype.storm.utils.Utils
The following examples show how to use
backtype.storm.utils.Utils. These examples are extracted from open source projects.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: jstorm Source File: ZookeeperManager.java License: Apache License 2.0 | 6 votes |
public static String getZKNodeData(String clusterName, String path) { String out = null; try { ClusterState clusterState = getAndCreateClusterState(clusterName); if (clusterState == null) { throw new IllegalStateException("Cluster state is null"); } byte[] data = clusterState.get_data(PathUtils.normalize_path(path), false); if (data != null && data.length > 0) { Object obj = Utils.maybe_deserialize(data); if (obj != null){ out = gson.toJson(obj); } else { out = new String(data); } } } catch (Exception e) { LOG.error("Get zookeeper data error!", e); } return out; }
Example 2
Source Project: jstorm Source File: rollback_topology.java License: Apache License 2.0 | 6 votes |
private static void rollbackTopology(String topologyName) { Map conf = Utils.readStormConfig(); NimbusClient client = NimbusClient.getConfiguredClient(conf); try { // update jar client.getClient().rollbackTopology(topologyName); CommandLineUtil.success("Successfully submit command rollback_topology " + topologyName); } catch (Exception e) { e.printStackTrace(); throw new RuntimeException(e); } finally { if (client != null) { client.close(); } } }
Example 3
Source Project: jstorm Source File: RocksTTLDBCache.java License: Apache License 2.0 | 6 votes |
@Override public Object get(String key) { for (Entry<Integer, ColumnFamilyHandle> entry : windowHandlers.entrySet()) { try { byte[] data = ttlDB.get(entry.getValue(), key.getBytes()); if (data != null) { try { return Utils.javaDeserialize(data); } catch (Exception e) { LOG.error("Failed to deserialize obj of " + key); ttlDB.remove(entry.getValue(), key.getBytes()); return null; } } } catch (Exception ignored) { } } return null; }
Example 4
Source Project: jstorm Source File: NettyUnitTest.java License: Apache License 2.0 | 6 votes |
private IConnection initNettyServer(int port) { ConcurrentHashMap<Integer, DisruptorQueue> deserializeQueues = new ConcurrentHashMap<Integer, DisruptorQueue>(); //ConcurrentHashMap<Integer, DisruptorQueue> deserializeCtrlQueues = new ConcurrentHashMap<Integer, DisruptorQueue>(); WaitStrategy wait = (WaitStrategy)Utils.newInstance("com.lmax.disruptor.TimeoutBlockingWaitStrategy", 5, TimeUnit.MILLISECONDS); DisruptorQueue recvControlQueue = DisruptorQueue.mkInstance("Dispatch-control", ProducerType.MULTI, 256, wait, false, 0, 0); Set<Integer> taskSet = new HashSet<Integer>(); taskSet.add(1); IConnection server = context.bind(null, port, deserializeQueues, recvControlQueue, true, taskSet); WaitStrategy waitStrategy = new BlockingWaitStrategy(); DisruptorQueue recvQueue = DisruptorQueue.mkInstance("NettyUnitTest", ProducerType.SINGLE, 1024, waitStrategy, false, 0, 0); server.registerQueue(task, recvQueue); return server; }
Example 5
Source Project: jstorm Source File: list.java License: Apache License 2.0 | 6 votes |
public static void main(String[] args) { NimbusClient client = null; try { Map conf = Utils.readStormConfig(); client = NimbusClient.getConfiguredClient(conf); if (args.length > 0 && !StringUtils.isBlank(args[0])) { String topologyName = args[0]; TopologyInfo info = client.getClient().getTopologyInfoByName(topologyName); System.out.println("Successfully get topology info \n" + Utils.toPrettyJsonString(info)); } else { ClusterSummary clusterSummary = client.getClient().getClusterInfo(); System.out.println("Successfully get cluster info \n" + Utils.toPrettyJsonString(clusterSummary)); } } catch (Exception e) { System.out.println(e.getMessage()); e.printStackTrace(); throw new RuntimeException(e); } finally { if (client != null) { client.close(); } } }
Example 6
Source Project: jstorm Source File: StormConfig.java License: Apache License 2.0 | 6 votes |
@SuppressWarnings("unchecked") public static Object readLocalObject(String topologyId, String readFile) throws IOException { String errMsg = "Failed to get topology configuration of " + topologyId + " file:" + readFile; byte[] bconf = FileUtils.readFileToByteArray(new File(readFile)); if (bconf == null) { errMsg += ", failed to read"; LOG.error(errMsg); throw new IOException(errMsg); } Object ret; try { ret = Utils.javaDeserialize(bconf); } catch (Exception e) { errMsg += ", failed to serialize the data"; LOG.error(errMsg); throw new IOException(errMsg); } return ret; }
Example 7
Source Project: storm-kafka-0.8-plus Source File: KafkaBoltTest.java License: Apache License 2.0 | 6 votes |
private boolean verifyMessage(String key, String message) { long lastMessageOffset = KafkaUtils.getOffset(simpleConsumer, kafkaConfig.topic, 0, OffsetRequest.LatestTime()) - 1; ByteBufferMessageSet messageAndOffsets = KafkaUtils.fetchMessages(kafkaConfig, simpleConsumer, new Partition(Broker.fromString(broker.getBrokerConnectionString()), 0), lastMessageOffset); MessageAndOffset messageAndOffset = messageAndOffsets.iterator().next(); Message kafkaMessage = messageAndOffset.message(); ByteBuffer messageKeyBuffer = kafkaMessage.key(); String keyString = null; String messageString = new String(Utils.toByteArray(kafkaMessage.payload())); if (messageKeyBuffer != null) { keyString = new String(Utils.toByteArray(messageKeyBuffer)); } assertEquals(key, keyString); assertEquals(message, messageString); return true; }
Example 8
Source Project: jstorm Source File: ZooKeeperDataViewTest.java License: Apache License 2.0 | 6 votes |
@BeforeClass public static void init() { String CONFIG_PATH = System.getProperty("user.home") + CONFIG_FILE; File file = new File(CONFIG_PATH); if (file.exists() == false) { SKIP = true; return; } try { zkobj = new Zookeeper(); System.getProperties().setProperty("storm.conf.file", CONFIG_PATH); Map conf = Utils.readStormConfig(); zk = zkobj.mkClient(conf, (List<String>) conf.get(Config.STORM_ZOOKEEPER_SERVERS), conf.get(Config.STORM_ZOOKEEPER_PORT), (String) conf.get(Config.STORM_ZOOKEEPER_ROOT)); gson = new GsonBuilder().setPrettyPrinting().create(); }catch(Throwable e) { e.printStackTrace(); SKIP = true; } }
Example 9
Source Project: jstorm Source File: Heartbeat.java License: Apache License 2.0 | 6 votes |
private void initSupervisorInfo(Map conf) { Set<Integer> portList = JStormUtils.getDefaultSupervisorPortList(conf); if (!StormConfig.local_mode(conf)) { try { boolean isLocalIP = myHostName.equals("127.0.0.1") || myHostName.equals("localhost"); if (isLocalIP) { throw new Exception("the hostname supervisor got is localhost"); } } catch (Exception e1) { LOG.error("get supervisor host error!", e1); throw new RuntimeException(e1); } supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf); } else { supervisorInfo = new SupervisorInfo(myHostName, supervisorId, portList, conf); } supervisorInfo.setVersion(Utils.getVersion()); String buildTs = Utils.getBuildTime(); supervisorInfo.setBuildTs(buildTs); LOG.info("jstorm version:{}, build ts:{}", supervisorInfo.getVersion(), supervisorInfo.getBuildTs()); }
Example 10
Source Project: storm-benchmark Source File: Grep.java License: Apache License 2.0 | 6 votes |
@Override public StormTopology getTopology(Config config) { final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM); final int matBoltNum = BenchmarkUtils.getInt(config, FM_NUM, DEFAULT_MAT_BOLT_NUM); final int cntBoltNum = BenchmarkUtils.getInt(config, CM_NUM, DEFAULT_CNT_BOLT_NUM); final String ptnString = (String) Utils.get(config, PATTERN_STRING, DEFAULT_PATTERN_STR); spout = new KafkaSpout(KafkaUtils.getSpoutConfig(config, new SchemeAsMultiScheme(new StringScheme()))); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout, spoutNum); builder.setBolt(FM_ID, new FindMatchingSentence(ptnString), matBoltNum) .localOrShuffleGrouping(SPOUT_ID); builder.setBolt(CM_ID, new CountMatchingSentence(), cntBoltNum) .fieldsGrouping(FM_ID, new Fields(FindMatchingSentence.FIELDS)); return builder.createTopology(); }
Example 11
Source Project: jstorm Source File: AckTransactionBolt.java License: Apache License 2.0 | 5 votes |
@Override public void emitDirect(int taskId, String streamId, Collection<Tuple> anchors, List<Object> tuple, ICollectorCallback callback) { if (anchors != null) { tuple.add(Utils.generateId(random)); } else { tuple.add(0l); } delegate.emitDirect(taskId, streamId, null, tuple, callback); }
Example 12
Source Project: jstorm Source File: FeederBatchSpout.java License: Apache License 2.0 | 5 votes |
@Override public boolean isReady(long txid) { if(!_waitToEmit) return true; List allBatches = (List) RegisteredGlobalState.getState(_id); if(allBatches.size() > _masterEmitted) { _masterEmitted++; return true; } else { Utils.sleep(2); return false; } }
Example 13
Source Project: jstorm Source File: WorkerAssignment.java License: Apache License 2.0 | 5 votes |
@Override public String toJSONString() { Map<String, String> map = new HashMap<>(); map.put(COMPONENTTONUM_TAG, Utils.to_json(componentToNum)); map.put(MEM_TAG, String.valueOf(mem)); map.put(CPU_TAG, String.valueOf(cpu)); map.put(HOSTNAME_TAG, hostName); map.put(JVM_TAG, jvm); map.put(NODEID_TAG, getNodeId()); map.put(PORT_TAG, String.valueOf(getPort())); return Utils.to_json(map); }
Example 14
Source Project: incubator-heron Source File: MultiStageAckingTopology.java License: Apache License 2.0 | 5 votes |
public void nextTuple() { // We explicitly slow down the spout to avoid the stream mgr to be the bottleneck Utils.sleep(1); final String word = words[rand.nextInt(words.length)]; // To enable acking, we need to emit tuple with MessageId, which is an object collector.emit(new Values(word), "MESSAGE_ID"); }
Example 15
Source Project: jstorm Source File: TridentBoltExecutor.java License: Apache License 2.0 | 5 votes |
private void updateTaskCounts(List<Integer> tasks) { synchronized (_currBatch) { if (_currBatch != null) { Map<Integer, Integer> taskEmittedTuples = _currBatch.taskEmittedTuples; for (Integer task : tasks) { int newCount = Utils.get(taskEmittedTuples, task, 0) + 1; taskEmittedTuples.put(task, newCount); } } } }
Example 16
Source Project: jstorm Source File: FastWordCountTopology.java License: Apache License 2.0 | 5 votes |
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) { _collector = collector; _rand = new Random(); sendingCount = 0; startTime = System.currentTimeMillis(); sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1); isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false); }
Example 17
Source Project: jstorm Source File: JStormHelper.java License: Apache License 2.0 | 5 votes |
@Override public <T> Object execute(T... args) { String topologyName = (String) args[0]; try { checkError(conf, topologyName); } catch (Exception e) { throw new RuntimeException(e); } Map stormConf = Utils.readStormConfig(); stormConf.putAll(conf); Map<String, Double> ret = JStormUtils.getMetrics(stormConf, topologyName, MetaType.TASK, null); for (Entry<String, Double> entry : ret.entrySet()) { String key = entry.getKey(); Double value = entry.getValue(); if (key.indexOf(MetricDef.FAILED_NUM) > 0) { Assert.assertTrue(key + " fail number should == 0", value == 0); } else if (key.indexOf(MetricDef.ACKED_NUM) > 0) { if (value == 0.0) { LOG.warn(key + ":" + value); } else { LOG.info(key + ":" + value); } } else if (key.indexOf(MetricDef.EMMITTED_NUM) > 0) { if (value == 0.0) { LOG.warn(key + ":" + value); } else { LOG.info(key + ":" + value); } } } return ret; }
Example 18
Source Project: jstorm Source File: WordSpout.java License: Apache License 2.0 | 5 votes |
public void nextTuple() { Utils.sleep(100); final String[] words = new String[] { "nathan", "mike", "jackson", "golda", "bertels" }; final Random rand = new Random(); final String word = words[rand.nextInt(words.length)]; _collector.emit(new Values(word)); }
Example 19
Source Project: jstorm Source File: TransactionCommon.java License: Apache License 2.0 | 5 votes |
public static Set<String> getUpstreamComponents(String componentId, TopologyContext context) { Set<String> upstreamComponents = new HashSet<>(); ComponentCommon componentCommon = Utils.getComponentCommon(context.getRawTopology(), componentId); Set<GlobalStreamId> input = componentCommon.get_inputs().keySet(); for (GlobalStreamId stream : input) { upstreamComponents.add(stream.get_componentId()); } return upstreamComponents; }
Example 20
Source Project: jstorm Source File: RotatingTransactionalState.java License: Apache License 2.0 | 5 votes |
public void cleanupBefore(long txid) { SortedMap<Long, Object> toDelete = _curr.headMap(txid); for(long tx: new HashSet<>(toDelete.keySet())) { _curr.remove(tx); try { _state.delete(txPath(tx)); } catch(RuntimeException e) { // Ignore NoNodeExists exceptions because when sync() it may populate _curr with stale data since // zookeeper reads are eventually consistent. if(!Utils.exceptionCauseIsInstanceOf(KeeperException.NoNodeException.class, e)) { throw e; } } } }
Example 21
Source Project: jstorm Source File: DefaultUncaughtExceptionHandler.java License: Apache License 2.0 | 5 votes |
@Override public void uncaughtException(Thread a, Throwable e) { try { Utils.handleUncaughtException(e); } catch (Error error) { LOG.info("Received error in main thread.. terminating server...", error); Runtime.getRuntime().exit(-2); } }
Example 22
Source Project: jstorm Source File: KeyRangeState.java License: Apache License 2.0 | 5 votes |
private void initKeyRangeState(int keyRange) { IRichCheckpointKvState<K, V, String> state = (IRichCheckpointKvState<K, V, String>) Utils.newInstance("com.alibaba.jstorm.hdfs.transaction.RocksDbHdfsState"); state.setStateName(context.getThisComponentId() + "/" + String.valueOf(keyRange)); state.init(context); keyRangeToState.put(keyRange, state); }
Example 23
Source Project: storm-benchmark Source File: PageViewCountTest.java License: Apache License 2.0 | 5 votes |
@Test public void componentParallelismCouldBeSetThroughConfig() { StormBenchmark benchmark = new PageViewCount(); Config config = new Config(); config.put(PageViewCount.SPOUT_NUM, 3); config.put(PageViewCount.VIEW_NUM, 4); config.put(PageViewCount.COUNT_NUM, 5); StormTopology topology = benchmark.getTopology(config); assertThat(topology).isNotNull(); TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.SPOUT_ID), 3); TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.VIEW_ID), 4); TestUtils.verifyParallelism(Utils.getComponentCommon(topology, PageViewCount.COUNT_ID), 5); }
Example 24
Source Project: jstorm Source File: StormZkClusterState.java License: Apache License 2.0 | 5 votes |
public void setObject(String path, Object obj) throws Exception { if (obj instanceof byte[]) { cluster_state.set_data(path, (byte[]) obj); } else if (obj instanceof String) { cluster_state.set_data(path, ((String) obj).getBytes()); } else { cluster_state.set_data(path, Utils.serialize(obj)); } }
Example 25
Source Project: jstorm Source File: UIUtils.java License: Apache License 2.0 | 5 votes |
public static Map<String, Object> getNimbusConf(String clusterName) { NimbusClient client = null; try { client = NimbusClientManager.getNimbusClient(clusterName); String jsonConf = client.getClient().getNimbusConf(); Map<String, Object> nimbusConf = (Map<String, Object>) Utils.from_json(jsonConf); return nimbusConf; } catch (Exception e) { NimbusClientManager.removeClient(clusterName); LOG.error(e.getMessage(), e); return UIUtils.readUiConfig(); } }
Example 26
Source Project: eagle Source File: UnitTopologyRunner.java License: Apache License 2.0 | 5 votes |
private void run(String topologyId, int numOfTotalWorkers, int numOfSpoutTasks, int numOfRouterBolts, int numOfAlertBolts, int numOfPublishExecutors, int numOfPublishTasks, Config config, boolean localMode) { backtype.storm.Config stormConfig = givenStormConfig == null ? new backtype.storm.Config() : givenStormConfig; // TODO: Configurable metric consumer instance number int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS) ? config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS; LOG.info("Set topology.message.timeout.secs as {}", messageTimeoutSecs); stormConfig.setMessageTimeoutSecs(messageTimeoutSecs); if (config.hasPath("metric")) { stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1); } stormConfig.setNumWorkers(numOfTotalWorkers); StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishExecutors, numOfPublishTasks, config).createTopology(); if (localMode) { LOG.info("Submitting as local mode"); LocalCluster cluster = new LocalCluster(); cluster.submitTopology(topologyId, stormConfig, topology); Utils.sleep(Long.MAX_VALUE); } else { LOG.info("Submitting as cluster mode"); try { StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology); } catch (Exception ex) { LOG.error("fail submitting topology {}", topology, ex); throw new IllegalStateException(ex); } } }
Example 27
Source Project: eagle Source File: MockStreamReceiver.java License: Apache License 2.0 | 5 votes |
/** * This unit test is not to mock the end2end logic of correlation spout, * but simply generate some sample data for following bolts testing */ @Override public void nextTuple() { PartitionedEvent event = MockSampleMetadataFactory.createRandomOutOfTimeOrderEventGroupedByName("sampleStream_1"); LOG.info("Receive {}", event); collector.emit(outputStreamIds.get( // group by the first field in event i.e. name (int) (event.getPartitionKey() % outputStreamIds.size())), Collections.singletonList(event)); Utils.sleep(500); }
Example 28
Source Project: jstorm Source File: AckTransactionSpout.java License: Apache License 2.0 | 5 votes |
@Override public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) { if (messageId != null) { addPendingTuple(currBatchId, streamId, messageId, tuple); tuple.add(Utils.generateId(random)); } else { // for non-anchor tuples, use 0 as default rootId tuple.add(0l); } return delegate.emit(streamId, tuple, null); }
Example 29
Source Project: eagle Source File: SampleClient2.java License: Apache License 2.0 | 5 votes |
/** * @param args */ public static void main(String[] args) { AtomicLong base1 = new AtomicLong(System.currentTimeMillis()); AtomicLong base2 = new AtomicLong(System.currentTimeMillis()); AtomicLong count = new AtomicLong(); Config config = ConfigFactory.load(); try (KafkaProducer<String, String> proceduer = SampleClient1.createProceduer(config)) { while (true) { nextUuid = String.format(instanceUuidTemp, UUID.randomUUID().toString()); nextReqId = String.format(reqIdTemp, UUID.randomUUID().toString()); int hostIndex = 6; for (int i = 0; i < hostIndex; i++) { sendMetric(base1, base2, count, proceduer, i); } if (count.get() % 600 == 0) { System.out.println("send 600 LOG/FAILURE metric!"); } Utils.sleep(3000); } } }
Example 30
Source Project: jstorm Source File: TridentBoltExecutor.java License: Apache License 2.0 | 5 votes |
@Override public void prepare(Map conf, TopologyContext context, OutputCollector collector) { _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L; _lastRotate = System.currentTimeMillis(); _batches = new RotatingMap<>(2); _context = context; _collector = collector; _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector)); _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector)); _coordConditions = (Map) context.getExecutorData("__coordConditions"); if (_coordConditions == null) { _coordConditions = new HashMap<>(); for (String batchGroup : _coordSpecs.keySet()) { CoordSpec spec = _coordSpecs.get(batchGroup); CoordCondition cond = new CoordCondition(); cond.commitStream = spec.commitStream; cond.expectedTaskReports = 0; for (String comp : spec.coords.keySet()) { CoordType ct = spec.coords.get(comp); if (ct.equals(CoordType.single())) { cond.expectedTaskReports += 1; } else { cond.expectedTaskReports += context.getComponentTasks(comp).size(); } } cond.targetTasks = new HashSet<>(); for (String component : Utils.get(context.getThisTargets(), COORD_STREAM(batchGroup), new HashMap<String, Grouping>()).keySet()) { cond.targetTasks.addAll(context.getComponentTasks(component)); } _coordConditions.put(batchGroup, cond); } context.setExecutorData("_coordConditions", _coordConditions); } _bolt.prepare(conf, context, _coordOutputCollector); }