Java Code Examples for com.alibaba.jstorm.utils.JStormUtils

The following are top voted examples for showing how to use com.alibaba.jstorm.utils.JStormUtils. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: learn_jstorm   File: Drpc.java   Source Code and License 6 votes vote down vote up
private THsHaServer initInvokeServer(Map conf, final Drpc service)
		throws Exception {
	int port = JStormUtils.parseInt(conf.get(Config.DRPC_INVOCATIONS_PORT));

	TNonblockingServerSocket socket = new TNonblockingServerSocket(port);
	THsHaServer.Args targsInvoke = new THsHaServer.Args(socket);
	targsInvoke.workerThreads(64);
	targsInvoke.protocolFactory(new TBinaryProtocol.Factory());
	targsInvoke
			.processor(new DistributedRPCInvocations.Processor<DistributedRPCInvocations.Iface>(
					service));

	THsHaServer invokeServer = new THsHaServer(targsInvoke);

	LOG.info("Successfully init Invoke Server " + port);
	return invokeServer;
}
 
Example 2
Project: jstorm-0.9.6.3-   File: NettyClientSync.java   Source Code and License 6 votes vote down vote up
/**
 * Don't take care of competition
 * 
 * @param blocked
 */
@SuppressWarnings("unchecked")
public void sendData() {
	sendTimer.start();
	try {
		MessageBatch batch = batchQueue.poll();
		if (batch == null) {

			disruptorQueue.consumeBatchWhenAvailable(this);

			batch = batchQueue.poll();
		}

		Channel channel = channelRef.get();
		flushBatch(batch, channel);
	} catch (Throwable e) {
		LOG.error("Occur e", e);
		String err = name + " nettyclient occur unknow exception";
		JStormUtils.halt_process(-1, err);
	} finally {
		sendTimer.stop();
	}
}
 
Example 3
Project: jstorm-0.9.6.3-   File: ClearThread.java   Source Code and License 6 votes vote down vote up
@Override
public void run() {

	for (Entry<String, Integer> e : drpcService.getIdtoStart().entrySet()) {
		if (TimeUtils.time_delta(e.getValue()) > REQUEST_TIMEOUT_SECS) {
			String id = e.getKey();
			
			drpcService.getIdtoResult().put(id, new DRPCExecutionException(
					"Request timed out"));
			Semaphore s = drpcService.getIdtoSem().get(id);
			if (s != null) {
				s.release();
			}
			drpcService.cleanup(id);
			LOG.info("Clear request " + id);
		}
	}
	
	JStormUtils.sleepMs(10);

}
 
Example 4
Project: jstrom   File: DelayStatusTransitionCallback.java   Source Code and License 6 votes vote down vote up
public int getDelaySeconds(Object[] args) {
    if (oldStatus != null && oldStatus.getDelaySecs() > 0) {
        return oldStatus.getDelaySecs();
    }

    Integer delaySecs = DelayStatusTransitionCallback.DEFAULT_DELAY_SECONDS;
    if (args == null || args.length == 0 || args[0] == null) {
        Map<?, ?> map = null;
        try {

            map = StormConfig.read_nimbus_topology_conf(data.getConf(), topologyid);
            delaySecs = JStormUtils.parseInt(map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), DEFAULT_DELAY_SECONDS);
        } catch (Exception e) {
            LOG.info("Failed to get topology configuration " + topologyid);
        }

    } else {
        delaySecs = JStormUtils.parseInt(args[0]);
    }

    if (delaySecs == null || delaySecs < 0) {
        delaySecs = DelayStatusTransitionCallback.DEFAULT_DELAY_SECONDS;
    }

    return delaySecs;
}
 
Example 5
Project: learn_jstorm   File: RefreshConnections.java   Source Code and License 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public RefreshConnections(WorkerData workerData, Set<Integer> outbound_tasks) {

	this.workerData = workerData;

	this.active = workerData.getActive();
	this.conf = workerData.getConf();
	this.zkCluster = workerData.getZkCluster();
	this.topologyId = workerData.getTopologyId();
	this.outboundTasks = outbound_tasks;
	this.nodeportSocket = workerData.getNodeportSocket();
	this.context = workerData.getContext();
	this.taskNodeport = workerData.getTaskNodeport();
	this.supervisorId = workerData.getSupervisorId();

	// this.endpoint_socket_lock = endpoint_socket_lock;
	frequence = JStormUtils.parseInt(
			conf.get(Config.TASK_REFRESH_POLL_SECS), 5);
	
	taskTimeoutSecs = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
	taskTimeoutSecs = taskTimeoutSecs*3;
}
 
Example 6
Project: jstorm-0.9.6.3-   File: JStackPage.java   Source Code and License 6 votes vote down vote up
public JStackPage() throws Exception {
	FacesContext ctx = FacesContext.getCurrentInstance();
	if (ctx.getExternalContext().getRequestParameterMap().get("host") != null) {
		host = ctx.getExternalContext().getRequestParameterMap()
				.get("host");
	}

	if (ctx.getExternalContext().getRequestParameterMap().get("port") != null) {

		port = JStormUtils.parseInt(ctx.getExternalContext()
				.getRequestParameterMap().get("port"), 0);
	}
	
	if (ctx.getExternalContext().getRequestParameterMap()
			.get(HttpserverUtils.HTTPSERVER_LOGVIEW_PARAM_WORKER_PORT) != null) {
		workerPort = JStormUtils.parseInt(ctx.getExternalContext()
				.getRequestParameterMap()
				.get(HttpserverUtils.HTTPSERVER_LOGVIEW_PARAM_WORKER_PORT));
	}

	init();
}
 
Example 7
Project: jstrom   File: FollowerRunnable.java   Source Code and License 6 votes vote down vote up
private void tryToBeLeader(final Map conf) throws Exception {
    boolean allowed = check_nimbus_priority();
    
    if (allowed){
        RunnableCallback masterCallback = new RunnableCallback() {
            @Override
            public void run() {
                try {
                    tryToBeLeader(conf);
                } catch (Exception e) {
                    LOG.error("To be master error", e);
                    JStormUtils.halt_process(30, "Cant't to be master" + e.getMessage());
                }
            }
        };
        LOG.info("This nimbus can be  leader");
        data.getStormClusterState().try_to_be_leader(Cluster.MASTER_SUBTREE, hostPort, masterCallback);
    }else {
    	LOG.info("This nimbus can't be leader");
    }
}
 
Example 8
Project: learn_jstorm   File: Heartbeat.java   Source Code and License 6 votes vote down vote up
/**
 * @param conf
 * @param stormClusterState
 * @param supervisorId
 * @param myHostName
 */
@SuppressWarnings({ "rawtypes", "unchecked" })
public Heartbeat(Map conf, StormClusterState stormClusterState,
		String supervisorId, AtomicBoolean active) {

	String myHostName = JStormServerUtils.getHostName(conf);

	this.stormClusterState = stormClusterState;
	this.supervisorId = supervisorId;
	this.conf = conf;
	this.myHostName = myHostName;
	this.startTime = TimeUtils.current_time_secs();
	this.active = active;
	this.frequence = JStormUtils.parseInt(conf
			.get(Config.SUPERVISOR_HEARTBEAT_FREQUENCY_SECS));

	initSupervisorInfo(conf);
	
	LOG.info("Successfully init supervisor heartbeat thread, " + supervisorInfo);
}
 
Example 9
Project: jstorm-0.9.6.3-   File: SyncProcessEvent.java   Source Code and License 6 votes vote down vote up
private String getGcDumpParam(Map totalConf) {
	// String gcPath = ConfigExtension.getWorkerGcPath(totalConf);
	String gcPath = JStormUtils.getLogDir();

	Date now = new Date();
	String nowStr = TimeFormat.getSecond(now);

	StringBuilder gc = new StringBuilder();

	gc.append(" -Xloggc:");
	gc.append(gcPath);
	gc.append(File.separator);
	gc.append("%TOPOLOGYID%-worker-%ID%-");
	gc.append(nowStr);
	gc.append("-gc.log -verbose:gc -XX:HeapDumpPath=");
	gc.append(gcPath).append(File.separator).append("java-%TOPOLOGYID%-")
		.append(nowStr).append(".hprof");
	gc.append(" ");

	return gc.toString();
}
 
Example 10
Project: learn_jstorm   File: SyncProcessEvent.java   Source Code and License 6 votes vote down vote up
private String getGcDumpParam(Map totalConf) {
	// String gcPath = ConfigExtension.getWorkerGcPath(totalConf);
	String gcPath = JStormUtils.getLogDir();

	Date now = new Date();
	String nowStr = TimeFormat.getSecond(now);

	StringBuilder gc = new StringBuilder();

	gc.append(" -Xloggc:");
	gc.append(gcPath);
	gc.append(File.separator);
	gc.append("%TOPOLOGYID%-worker-%ID%-");
	gc.append(nowStr);
	gc.append("-gc.log -verbose:gc -XX:HeapDumpPath=");
	gc.append(gcPath).append(File.separator).append("java-%TOPOLOGYID%-")
		.append(nowStr).append(".hprof");
	gc.append(" ");

	return gc.toString();
}
 
Example 11
Project: jstrom   File: TopologyHistory.java   Source Code and License 6 votes vote down vote up
@Override
public Object fromKV(byte[] key, byte[] value) {
    String[] keyParts = new String(key).split(MetricUtils.DELIM);
    long time = 0;
    if (keyParts.length >= 3) {
        this.clusterName = keyParts[0];
        this.topologyName = keyParts[1];
        time = Long.valueOf(keyParts[2]);
    }

    String[] valueParts = new String(value).split(MetricUtils.DELIM);
    if (valueParts.length >= 3) {
        this.topologyId = valueParts[0];
        String tag = valueParts[1];
        if (tag.equals(KVSerializable.START)) {
            this.start = new Date(time);
        } else {
            this.end = new Date(time);
        }
        this.sampleRate = JStormUtils.parseDouble(valueParts[2], 0.1d);
    }

    return this;
}
 
Example 12
Project: jstorm-0.9.6.3-   File: CgroupManager.java   Source Code and License 6 votes vote down vote up
public void shutDownWorker(String workerId, boolean isKilled) {
	CgroupCommon workerGroup = new CgroupCommon(workerId, h,
			this.rootCgroup);
	try {
		if (isKilled == false) {
			for (Integer pid : workerGroup.getTasks()) {
				JStormUtils.kill(pid);
			}
			JStormUtils.sleepMs(1500);
		}
		center.delete(workerGroup);
	}catch(Exception e) {
		LOG.info("No task of " + workerId);
	}
	
}
 
Example 13
Project: jstrom   File: LocalCluster.java   Source Code and License 6 votes vote down vote up
@Override
public void submitTopologyWithOpts(String topologyName, Map conf, StormTopology topology, SubmitOptions submitOpts) {
    // TODO Auto-generated method stub
    if (!Utils.isValidConf(conf))
        throw new RuntimeException("Topology conf is not json-serializable");
    JStormUtils.setLocalMode(true);
    conf.put(Config.STORM_CLUSTER_MODE, "local");

    try {
        if (submitOpts == null) {
            state.getNimbus().submitTopology(topologyName, null, Utils.to_json(conf), topology);
        } else {
            state.getNimbus().submitTopologyWithOpts(topologyName, null, Utils.to_json(conf), topology, submitOpts);
        }

    } catch (Exception e) {
        // TODO Auto-generated catch block
        LOG.error("Failed to submit topology " + topologyName, e);
        throw new RuntimeException(e);
    }
}
 
Example 14
Project: jstorm-0.9.6.3-   File: RotatingMapTrigger.java   Source Code and License 6 votes vote down vote up
public RotatingMapTrigger(Map conf, String name, DisruptorQueue queue) {
	this.name = name;
	this.queue = queue;

	int msgTimeOut = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
	frequence = (msgTimeOut) / (Acker.TIMEOUT_BUCKET_NUM - 1);
	if (frequence <= 0) {
		frequence = 1;
	}

	firstTime = JStormUtils.parseInt(
			conf.get(Config.SUPERVISOR_WORKER_START_TIMEOUT_SECS), 120);

	firstTime += frequence;
}
 
Example 15
Project: jstrom   File: FilesController.java   Source Code and License 6 votes vote down vote up
private void parseString(String input) {
    Map<String, Map> map = (Map<String, Map>) JStormUtils
            .from_json(input);
    for (Map jobj : map.values()) {
        FileAttribute attribute = FileAttribute.fromJSONObject(jobj);
        if (attribute != null) {

            if (JStormUtils.parseBoolean(attribute.getIsDir(), false) == true) {
                dirs.add(attribute);
            } else {
                files.add(attribute);
            }

        }

    }
}
 
Example 16
Project: jstorm-0.9.6.3-   File: MetaSimpleClient.java   Source Code and License 6 votes vote down vote up
public void initMetaConsumer() throws MQClientException {
	LOG.info("MetaSpoutConfig:" + metaSpoutConfig);
	
	consumer = new MetaPullConsumer(metaSpoutConfig.getConsumerGroup());
	consumer.setInstanceName(taskIndex + "." + JStormUtils.process_pid());
	
	if (metaSpoutConfig.getNameServer() != null) {
		// this is for alipay
		if (nameServer == null) {
			nameServer = metaSpoutConfig.getNameServer();
			
			System.setProperty("rocketmq.namesrv.domain",
					metaSpoutConfig.getNameServer());
		}else if (metaSpoutConfig.getNameServer().equals(nameServer) == false) {
			throw new RuntimeException("Different nameserver address in the same worker "
					+ nameServer + ":" + metaSpoutConfig.getNameServer());
			
		}
	}

       consumer.start();
       
       LOG.info("Successfully start meta consumer");
}
 
Example 17
Project: jstrom   File: TaskHeartbeatTrigger.java   Source Code and License 6 votes vote down vote up
public TaskHeartbeatTrigger(Map conf, String name, DisruptorQueue queue, BlockingQueue<Object> controlQueue, int taskId, String componentId,
        TopologyContext sysTopologyCtx, ITaskReportErr reportError) {
    this.name = name;
    this.queue = queue;
    this.controlQueue = controlQueue;
    this.opCode = TimerConstants.TASK_HEARTBEAT;

    this.taskId = taskId;
    this.componentId = componentId;
    this.sysTopologyCtx = sysTopologyCtx;

    this.frequence = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
    this.firstTime = frequence;

    this.executeThreadHbTime = TimeUtils.current_time_secs();
    this.taskHbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_TIMEOUT_SECS), 180);
    this.intervalCheck = new IntervalCheck();
    this.intervalCheck.setInterval(taskHbTimeout);
    this.intervalCheck.start();

    this.reportError = reportError;

    this.uptime = new UptimeComputer();
}
 
Example 18
Project: jstrom   File: TimeoutMemCache.java   Source Code and License 6 votes vote down vote up
@Override
public void init(Map<Object, Object> conf) {
    this.defaultTimeout = ConfigExtension.getDefaultCacheTimeout(conf);
    registerCacheWindow(defaultTimeout);

    List<Object> list = (List) ConfigExtension.getCacheTimeoutList(conf);
    if (list != null) {
        for (Object obj : list) {
            Integer timeoutSecond = JStormUtils.parseInt(obj);
            if (timeoutSecond == null) {
                continue;
            }
            registerCacheWindow(timeoutSecond);
        }
    }
}
 
Example 19
Project: jstorm-0.9.6.3-   File: MetaSimpleClient.java   Source Code and License 6 votes vote down vote up
protected Set<MessageQueue> getMQ() throws MQClientException {
	Set<MessageQueue> mqs = consumer.
			fetchSubscribeMessageQueues(metaSpoutConfig.getTopic());
	if (taskParallel > mqs.size()) {
		throw new RuntimeException("Two much task to fetch " + metaSpoutConfig);
	}
	
	List<MessageQueue> mqList = JStormUtils.mk_list(mqs);
	Set<MessageQueue> ret = new HashSet<MessageQueue>();
	for (int i = taskIndex; i < mqList.size(); i += taskParallel) {
		ret.add(mqList.get(i));
	}
	
	if (ret.size() == 0) {
		throw new RuntimeException("No meta queue need to be consume");
	}
	return ret;
}
 
Example 20
Project: jstrom   File: TransactionalWordsTest.java   Source Code and License 6 votes vote down vote up
@Test
public void test_transaction_word() {
    try {
        MemoryTransactionalSpout spout = new MemoryTransactionalSpout(DATA, new Fields("word"), PARTITION_TAKE_PER_BATCH);
        TransactionalTopologyBuilder builder = new TransactionalTopologyBuilder("top-n-words", "spout", spout, 2);
        builder.setBolt("count", new KeyedCountUpdater(), 5).fieldsGrouping("spout", new Fields("word"));
        builder.setBolt("bucketize", new Bucketize()).shuffleGrouping("count");
        builder.setBolt("buckets", new BucketCountUpdater(), 5).fieldsGrouping("bucketize", new Fields("bucket"));

        LocalCluster cluster = new LocalCluster();

        Config config = new Config();
        config.setDebug(true);
        config.setMaxSpoutPending(3);

        cluster.submitTopology("top-n-topology", config, builder.buildTopology());

        JStormUtils.sleepMs(60 * 1000);
        cluster.shutdown();
    } catch (Exception e) {
        Assert.fail("Failed to run simple transaction");
    }

}
 
Example 21
Project: learn_jstorm   File: CgroupManager.java   Source Code and License 6 votes vote down vote up
public void shutDownWorker(String workerId, boolean isKilled) {
	CgroupCommon workerGroup = new CgroupCommon(workerId, h,
			this.rootCgroup);
	try {
		if (isKilled == false) {
			for (Integer pid : workerGroup.getTasks()) {
				JStormUtils.kill(pid);
			}
			JStormUtils.sleepMs(1500);
		}
		center.delete(workerGroup);
	}catch(Exception e) {
		LOG.info("No task of " + workerId);
	}
	
}
 
Example 22
Project: jstrom   File: MetricMeta.java   Source Code and License 6 votes vote down vote up
@Override
public Object fromKV(byte[] key, byte[] value) {
    String[] keyParts = new String(key).split(MetricUtils.DELIM);
    if (keyParts.length >= 4) {
        this.clusterName = keyParts[0];
        this.topologyId = keyParts[1];
        this.metaType = Integer.valueOf(keyParts[2]);
        this.id = Long.valueOf(keyParts[3]);
        this.sid = this.id + "";
    }
    String[] valueParts = new String(value).split(MetricUtils.DELIM);
    if (valueParts.length >= 8) {
        this.component = valueParts[0];
        this.taskId = JStormUtils.parseInt(valueParts[1], 0);
        this.streamId = valueParts[2];
        this.metricType = JStormUtils.parseInt(valueParts[3], 0);
        this.host = valueParts[4];
        this.port = JStormUtils.parseInt(valueParts[5], 0);
        this.metricGroup = valueParts[6];
        this.metricName = valueParts[7];
    }
    return this;
}
 
Example 23
Project: jstrom   File: NimbusData.java   Source Code and License 6 votes vote down vote up
public void createFileHandler() {
    TimeCacheMap.ExpiredCallback<Object, Object> expiredCallback = new TimeCacheMap.ExpiredCallback<Object, Object>() {
        @Override
        public void expire(Object key, Object val) {
            try {
                LOG.info("Close file " + String.valueOf(key));
                if (val != null) {
                    if (val instanceof Channel) {
                        Channel channel = (Channel) val;
                        channel.close();
                    } else if (val instanceof BufferFileInputStream) {
                        BufferFileInputStream is = (BufferFileInputStream) val;
                        is.close();
                    }
                }
            } catch (IOException e) {
                LOG.error(e.getMessage(), e);
            }

        }
    };

    int file_copy_expiration_secs = JStormUtils.parseInt(conf.get(Config.NIMBUS_FILE_COPY_EXPIRATION_SECS), 30);
    uploaders = new TimeCacheMap<Object, Object>(file_copy_expiration_secs, expiredCallback);
    downloaders = new TimeCacheMap<Object, Object>(file_copy_expiration_secs, expiredCallback);
}
 
Example 24
Project: jstrom   File: SyncProcessEvent.java   Source Code and License 6 votes vote down vote up
private String getGcDumpParam(String topologyName, Map totalConf) {
    // String gcPath = ConfigExtension.getWorkerGcPath(totalConf);
    String gcPath = JStormUtils.getLogDir();

    Date now = new Date();
    String nowStr = TimeFormat.getSecond(now);

    StringBuilder gc = new StringBuilder(256);
    gc.append(" -Xloggc:");
    gc.append(gcPath).append(File.separator);
    gc.append(topologyName).append(File.separator);
    gc.append("%TOPOLOGYID%-worker-%ID%");
    gc.append("-gc.log -verbose:gc -XX:HeapDumpPath=");
    gc.append(gcPath).append(File.separator).append(topologyName).append(File.separator).append("java-%TOPOLOGYID%-").append(nowStr).append(".hprof");
    gc.append(" ");


    return gc.toString();
}
 
Example 25
Project: jstorm-0.9.6.3-   File: SequenceSpout.java   Source Code and License 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	this.collector = collector;

	if (conf.get("spout.max.sending.num") == null) {
		isLimited = false;
	} else {
		isLimited = true;
		SPOUT_MAX_SEND_NUM = JStormUtils.parseLong(conf
				.get("spout.max.sending.num"));
	}

	isFinished = false;

	tpsCounter = new TpsCounter(context.getThisComponentId() + ":"
			+ context.getThisTaskId());

	MAX_PENDING_COUNTER = getMaxPending(conf);
	
	bufferLen = JStormUtils.parseInt(conf.get("byte.buffer.len"), 0);
	
	random = new Random();
	random.setSeed(System.currentTimeMillis());
	
	LOG.info("Finish open, buffer Len:"  + bufferLen);

}
 
Example 26
Project: jstrom   File: BoltCollector.java   Source Code and License 5 votes vote down vote up
public static void put_xor(Map<Long, Long> pending, Long key, Long id) {
    // synchronized (pending) {
    Long curr = pending.get(key);
    if (curr == null) {
        curr = 0L;
    }
    pending.put(key, JStormUtils.bit_xor(curr, id));
    // }
}
 
Example 27
Project: jstrom   File: ConfigExtension.java   Source Code and License 5 votes vote down vote up
public static Integer getSpoutParallelism(Map conf, String componentName) {
    Integer ret = null;
    Map<String, String> map = (Map<String, String>) (conf.get(SPOUT_PARALLELISM));
    if (map != null)
        ret = JStormUtils.parseInt(map.get(componentName));
    return ret;
}
 
Example 28
Project: learn_jstorm   File: SpoutOutput.java   Source Code and License 5 votes vote down vote up
public void setValues(String stream, Long emitted, Double sendTps,
		Double process, Long acked, Long failed) {
	this.stream = stream;
	this.emitted = JStormUtils.formatValue(emitted);
	this.sendTps = JStormUtils.formatValue(sendTps);
	this.acked = JStormUtils.formatValue(acked);
	this.failed = JStormUtils.formatValue(failed);
	this.process = JStormUtils.formatValue(process);

}
 
Example 29
Project: learn_jstorm   File: ConfigExtension.java   Source Code and License 5 votes vote down vote up
public static boolean getTopologyBufferSizeLimited(Map conf) {
	boolean isSynchronized = isNettySyncMode(conf);
	if (isSynchronized == true) {
		return true;
	}
	
	return JStormUtils.parseBoolean(conf.get(TOPOLOGY_BUFFER_SIZE_LIMITED), true);
	
}
 
Example 30
Project: jstrom   File: ConfigExtension.java   Source Code and License 5 votes vote down vote up
public static double getSupervisorSlotsPortCpuWeight(Map conf) {
    Object value = conf.get(SUPERVISOR_SLOTS_PORT_CPU_WEIGHT);
    Double ret = JStormUtils.convertToDouble(value);
    if (ret == null || ret <= 0) {
        return 1.2;
    } else {
        return ret;
    }
}
 
Example 31
Project: jstorm-0.9.6.3-   File: ComponentInput.java   Source Code and License 5 votes vote down vote up
public void setValues(GlobalStreamId gstreamId, Double recvTps,
		Double process, Long acked, Long failed) {
	this.componentId = gstreamId.get_componentId();
	this.stream = gstreamId.get_streamId();
	this.recvTps = JStormUtils.formatValue(recvTps);
	this.acked = JStormUtils.formatValue(acked);
	this.failed = JStormUtils.formatValue(failed);
	this.process = JStormUtils.formatValue(process);

}
 
Example 32
Project: jstrom   File: SimpleBatchTopology.java   Source Code and License 5 votes vote down vote up
public static TopologyBuilder SetBuilder() {
	BatchTopologyBuilder topologyBuilder = new BatchTopologyBuilder(
			topologyName);
	
	int spoutParallel = JStormUtils.parseInt(conf.get("topology.spout.parallel"), 1);

	BoltDeclarer boltDeclarer = topologyBuilder.setSpout("Spout",
			new SimpleSpout(), spoutParallel);

	int boltParallel = JStormUtils.parseInt(conf.get("topology.bolt.parallel"), 2);
	topologyBuilder.setBolt("Bolt", new SimpleBolt(), boltParallel).shuffleGrouping(
			"Spout");

	return topologyBuilder.getTopologyBuilder();
}
 
Example 33
Project: learn_jstorm   File: AckObject.java   Source Code and License 5 votes vote down vote up
public void update_ack(Object value) {
	synchronized (this) {
		if (val == null) {
			val = Long.valueOf(0);
		}
		val = JStormUtils.bit_xor(val, value);
	}
}
 
Example 34
Project: jstorm-0.9.6.3-   File: NimbusUtils.java   Source Code and License 5 votes vote down vote up
public static Integer componentParalism(Map stormConf,
		ComponentCommon common) {
	Map mergeMap = new HashMap();
	mergeMap.putAll(stormConf);

	String jsonConfString = common.get_json_conf();
	if (jsonConfString != null) {
		Map componentMap = (Map) JStormUtils.from_json(jsonConfString);
		mergeMap.putAll(componentMap);
	}

	Integer taskNum = common.get_parallelism_hint();
	if (taskNum == null) {
		taskNum = Integer.valueOf(1);
	}

	// don't get taskNum from component configuraiton
	// skip .setTaskNum
	// Integer taskNum = null;
	// Object taskNumObject = mergeMap.get(Config.TOPOLOGY_TASKS);
	// if (taskNumObject != null) {
	// taskNum = JStormUtils.parseInt(taskNumObject);
	// } else {
	// taskNum = common.get_parallelism_hint();
	// if (taskNum == null) {
	// taskNum = Integer.valueOf(1);
	// }
	// }

	Object maxTaskParalismObject = mergeMap
			.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
	if (maxTaskParalismObject == null) {
		return taskNum;
	} else {
		int maxTaskParalism = JStormUtils.parseInt(maxTaskParalismObject);

		return Math.min(maxTaskParalism, taskNum);
	}

}
 
Example 35
Project: jstorm-0.9.6.3-   File: NimbusServer.java   Source Code and License 5 votes vote down vote up
private void initMonitor(Map conf) {
	final ScheduledExecutorService scheduExec = data.getScheduExec();

	// Schedule Nimbus monitor
	MonitorRunnable r1 = new MonitorRunnable(data);

	int monitor_freq_secs = JStormUtils.parseInt(
			conf.get(Config.NIMBUS_MONITOR_FREQ_SECS), 10);
	scheduExec.scheduleAtFixedRate(r1, 0, monitor_freq_secs,
			TimeUnit.SECONDS);

	LOG.info("Successfully init Monitor thread");
}
 
Example 36
Project: jstrom   File: SyncContainerHb.java   Source Code and License 5 votes vote down vote up
public void handleWriteDir() {
    if (StringUtils.isBlank(writeDir) == true) {
        return;
    }

    String seconds = String.valueOf(System.currentTimeMillis() / SECOND_MILLISCOND);

    String path = writeDir + File.separator + seconds;

    try {
        PathUtils.touch(path);
        LOG.info("Successfully touch " + path);
    } catch (IOException e) {
        LOG.error("Failed to touch " + path, e);
        throw new RuntimeException("Failed to touch " + path);
    }

    File file = new File(writeDir);
    String[] files = file.list();
    if (files.length == 0) {
        LOG.info(readDir + " doesn't contain hearbeat files right now");
        return;
    }

    List<String> fileList = JStormUtils.mk_list(files);
    Collections.sort(fileList);

    removeOld(fileList, writeDir);

}
 
Example 37
Project: learn_jstorm   File: NimbusUtils.java   Source Code and License 5 votes vote down vote up
public static Integer componentParalism(Map stormConf, ComponentCommon common) {
    Map mergeMap = new HashMap();
    mergeMap.putAll(stormConf);

    String jsonConfString = common.get_json_conf();
    if (jsonConfString != null) {
        Map componentMap = (Map) JStormUtils.from_json(jsonConfString);
        mergeMap.putAll(componentMap);
    }

    Integer taskNum = common.get_parallelism_hint();
    if (taskNum == null) {
        taskNum = Integer.valueOf(1);
    }

    // don't get taskNum from component configuraiton
    // skip .setTaskNum
    // Integer taskNum = null;
    // Object taskNumObject = mergeMap.get(Config.TOPOLOGY_TASKS);
    // if (taskNumObject != null) {
    // taskNum = JStormUtils.parseInt(taskNumObject);
    // } else {
    // taskNum = common.get_parallelism_hint();
    // if (taskNum == null) {
    // taskNum = Integer.valueOf(1);
    // }
    // }

    Object maxTaskParalismObject = mergeMap.get(Config.TOPOLOGY_MAX_TASK_PARALLELISM);
    if (maxTaskParalismObject == null) {
        return taskNum;
    } else {
        int maxTaskParalism = JStormUtils.parseInt(maxTaskParalismObject);

        return Math.min(maxTaskParalism, taskNum);
    }

}
 
Example 38
Project: jstrom   File: Common.java   Source Code and License 5 votes vote down vote up
/**
 * Check Whether ID of Bolt or spout is system_id
 * 
 * @param topology
 * @throws InvalidTopologyException
 */
@SuppressWarnings("unchecked")
public static void validate_ids(StormTopology topology, String topologyId) throws InvalidTopologyException {
    String topologyName = topologyIdToName(topologyId);
    if (!charValidate(topologyName)) {
        throw new InvalidTopologyException(topologyName + " is not a valid topology name. " + nameErrorInfo);
    }

    List<String> list = new ArrayList<String>();

    for (StormTopology._Fields field : Thrift.STORM_TOPOLOGY_FIELDS) {
        Object value = topology.getFieldValue(field);
        if (value != null) {
            Map<String, Object> obj_map = (Map<String, Object>) value;

            Set<String> commids = obj_map.keySet();

            for (String id : commids) {
                if (system_id(id) || !charComponentValidate(id)) {
                    throw new InvalidTopologyException(id + " is not a valid component id. " + compErrorInfo);
                }
            }

            for (Object obj : obj_map.values()) {
                validate_component(obj);
            }

            list.addAll(commids);
        }
    }

    List<String> offending = JStormUtils.getRepeat(list);
    if (offending.isEmpty() == false) {
        throw new InvalidTopologyException("Duplicate component ids: " + offending);
    }

}
 
Example 39
Project: learn_jstorm   File: Common.java   Source Code and License 5 votes vote down vote up
public static StormTopology add_system_components(StormTopology topology) {
	// generate inputs
	Map<GlobalStreamId, Grouping> inputs = new HashMap<GlobalStreamId, Grouping>();

	// generate outputs
	HashMap<String, StreamInfo> outputs = new HashMap<String, StreamInfo>();
	ArrayList<String> fields = new ArrayList<String>();

	outputs.put(Constants.SYSTEM_TICK_STREAM_ID,
			Thrift.outputFields(JStormUtils.mk_list("rate_secs")));
	outputs.put(Constants.METRICS_TICK_STREAM_ID,
			Thrift.outputFields(JStormUtils.mk_list("interval")));
	outputs.put(Constants.CREDENTIALS_CHANGED_STREAM_ID,
			Thrift.outputFields(JStormUtils.mk_list("creds")));

	ComponentCommon common = new ComponentCommon(inputs, outputs);

	IBolt ackerbolt = new SystemBolt();

	Bolt bolt = Thrift.mkBolt(inputs, ackerbolt, outputs,
			Integer.valueOf(0));

	topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, bolt);

	add_system_streams(topology);

	return topology;

}
 
Example 40
Project: jstorm-0.9.6.3-   File: Httpserver.java   Source Code and License 5 votes vote down vote up
public LogHandler(Map conf) {

			logDir = JStormUtils.getLogDir();
			String logDirPath = PathUtils.getCanonicalPath(logDir);
			if (logDirPath == null) {
				accessDirs.add(logDir);
			}else {
				accessDirs.add(logDirPath);
			}
			
            stormHome = System.getProperty("jstorm.home");
            if (stormHome != null) {
	            String stormHomePath = PathUtils.getCanonicalPath(stormHome);
	            if (stormHomePath == null) {
					accessDirs.add(stormHome);
				}else {
					accessDirs.add(stormHomePath);
				}
            }
            
            String confDir = System.getProperty(Constants.JSTORM_CONF_DIR);
            if (StringUtils.isBlank(confDir) == false) {
            	String confDirPath = PathUtils.getCanonicalPath(confDir);
            	if (confDirPath != null) {
            		accessDirs.add(confDirPath);
            	}
            }
            
            
			this.conf = conf;

			LOG.info("logview logDir=" + logDir); // +++

		}
 
Example 41
Project: jstrom   File: SyncProcessEvent.java   Source Code and License 5 votes vote down vote up
public String getLogParameter(Map conf, String stormHome, String topologyName, int port) {
    final String LOGBACK_CONF_TAG = "logback.configurationFile";
    final String LOGBACK_CONF_TAG_CMD = " -D" + LOGBACK_CONF_TAG + "=";
    final String DEFAULT_LOG_CONF = "jstorm.logback.xml";

    String logFileName = JStormUtils.genLogName(topologyName, port);
    // String logFileName = topologyId + "-worker-" + port + ".log";


    StringBuilder commandSB = new StringBuilder();
    commandSB.append(" -Dlogfile.name=");
    commandSB.append(logFileName);
    commandSB.append(" -Dtopology.name=").append(topologyName);

    // commandSB.append(" -Dlog4j.ignoreTCL=true");

    String userDefLogbackConf = ConfigExtension.getUserDefinedLogbackConf(conf);
    String logConf = System.getProperty(LOGBACK_CONF_TAG);

    if (StringUtils.isBlank(userDefLogbackConf) == false) {
        LOG.info("Use user fined back conf " + userDefLogbackConf);
        commandSB.append(LOGBACK_CONF_TAG_CMD).append(userDefLogbackConf);
    } else if (StringUtils.isBlank(logConf) == false) {
        commandSB.append(LOGBACK_CONF_TAG_CMD).append(logConf);
    } else if (StringUtils.isBlank(stormHome) == false) {
        commandSB.append(LOGBACK_CONF_TAG_CMD).append(stormHome).append(File.separator).append("conf").append(File.separator).append(DEFAULT_LOG_CONF);
    } else {
        commandSB.append(LOGBACK_CONF_TAG_CMD + DEFAULT_LOG_CONF);
    }

    final String LOG4J_CONF_TAG = "log4j.configuration";
    String userDefLog4jConf = ConfigExtension.getUserDefinedLog4jConf(conf);
    if (StringUtils.isBlank(userDefLog4jConf) == false) {
        LOG.info("Use user fined log4j conf " + userDefLog4jConf);
        commandSB.append(" -D" + LOG4J_CONF_TAG + "=").append(userDefLog4jConf);
    }

    return commandSB.toString();
}
 
Example 42
Project: learn_jstorm   File: MkLocalShuffer.java   Source Code and License 5 votes vote down vote up
public List<Integer> grouper(List<Object> values) {
	int index = getActiveTask(randomrange, outTasks);
	// If none active tasks were found, still send message to a task
	if (index == -1)
	    index = randomrange.nextInt();

	return JStormUtils.mk_list(outTasks.get(index));
}
 
Example 43
Project: learn_jstorm   File: LocalCluster.java   Source Code and License 5 votes vote down vote up
@Override
public void shutdown() {
	// TODO Auto-generated method stub
	// in order to avoid kill topology's command competition
	// it take 10 seconds to remove topology's node
	JStormUtils.sleepMs(10 * 1000);
	this.state.clean();
}
 
Example 44
Project: jstrom   File: Drpc.java   Source Code and License 5 votes vote down vote up
private THsHaServer initInvokeServer(Map conf, final Drpc service) throws Exception {
    int port = JStormUtils.parseInt(conf.get(Config.DRPC_INVOCATIONS_PORT));

    TNonblockingServerSocket socket = new TNonblockingServerSocket(port);
    THsHaServer.Args targsInvoke = new THsHaServer.Args(socket);
    targsInvoke.workerThreads(64);
    targsInvoke.protocolFactory(new TBinaryProtocol.Factory());
    targsInvoke.processor(new DistributedRPCInvocations.Processor<DistributedRPCInvocations.Iface>(service));

    THsHaServer invokeServer = new THsHaServer(targsInvoke);

    LOG.info("Successfully init Invoke Server " + port);
    return invokeServer;
}
 
Example 45
Project: learn_jstorm   File: BatchSpoutTrigger.java   Source Code and License 5 votes vote down vote up
/**
 * @throws Exception
 * 
 */
public void initMsgId() throws Exception {
	Long zkMsgId = null;
	byte[] data = zkClient.get_data(ZK_NODE_PATH, false);
	if (data != null) {
		String value = new String(data);
		try {
			zkMsgId = Long.valueOf(value);
			LOG.info("ZK msgId:" + zkMsgId);
		} catch (Exception e) {
			LOG.warn("Failed to get msgId ", e);

		}

	}

	if (zkMsgId != null) {
		BatchId.updateId(zkMsgId);
	}

	int max_spout_pending = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING), 1);

	for (int i = 0; i < max_spout_pending; i++) {
		BatchSpoutMsgId msgId = BatchSpoutMsgId.mkInstance();
		if (currentBatchId == null) {
			currentBatchId = msgId.getBatchId();
		}
		batchQueue.offer(msgId);
		LOG.info("Push into queue," + msgId);
	}

}
 
Example 46
Project: jstorm-0.9.6.3-   File: DBBolt.java   Source Code and License 5 votes vote down vote up
@Override
public void run() {
	LOG.info("Begin to run revert thread");
	isRunRevert.set(true);

	while (isRunRevert.get() == true) {
		CommitedValue commitedValue = null;
		try {
			commitedValue = revertQueue.take();
		} catch (InterruptedException e1) {
		}
		if (commitedValue == null) {
			continue;
		}

		try {
			commitedValue.revert();
			LOG.info("Successfully revert " + commitedValue);
		} catch (Exception e) {
			LOG.info("Failed to do revert " + commitedValue);
			JStormUtils.sleepMs(100);
			revertQueue.offer(commitedValue);
		}
	}

	LOG.info("Successfully quit revert thread");
}
 
Example 47
Project: jstrom   File: CountBolt.java   Source Code and License 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context) {
	this.conf = stormConf;

	int timeoutSeconds = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
	counters = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
	
	LOG.info("Successfully do prepare");

}
 
Example 48
Project: jstorm-0.9.6.3-   File: StatFunction.java   Source Code and License 5 votes vote down vote up
public static void incr_val(Map<Object, Long> map, Object key, Long amt) {
	Long value = Long.valueOf(0);
	if (map.containsKey(key)) {
		value = map.get(key);
	}
	value = (Long) JStormUtils.add(value, amt);
	map.put(key, value);
}
 
Example 49
Project: jstorm-0.9.6.3-   File: DelayStatusTransitionCallback.java   Source Code and License 5 votes vote down vote up
public int getDelaySeconds(Object[] args) {
	if (oldStatus != null && oldStatus.getDelaySecs() > 0) {
		return oldStatus.getDelaySecs();
	}
	
	Integer delaySecs = DelayStatusTransitionCallback.DEFAULT_DELAY_SECONDS;
	if (args == null || args.length == 0 || args[0] == null) {
		Map<?, ?> map = null;
		try {

			map = StormConfig.read_nimbus_topology_conf(data.getConf(),
					topologyid);
			delaySecs = JStormUtils.parseInt(
					map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 
					DEFAULT_DELAY_SECONDS);
		} catch (Exception e) {
			LOG.info("Failed to get topology configuration " + topologyid);
		}

	} else {
		delaySecs = JStormUtils.parseInt(args[0]);
	}

	if (delaySecs == null || delaySecs < 0) {
		delaySecs = DelayStatusTransitionCallback.DEFAULT_DELAY_SECONDS;
	}
	
	return delaySecs;
}
 
Example 50
Project: jstrom   File: Acker.java   Source Code and License 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    // pending = new TimeCacheMap<Object, AckObject>(timeoutSec,
    // TIMEOUT_BUCKET_NUM);
    this.pending = new RotatingMap<Object, AckObject>(TIMEOUT_BUCKET_NUM);
    this.rotateTime = 1000L * JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30) / (TIMEOUT_BUCKET_NUM - 1);
}
 
Example 51
Project: learn_jstorm   File: Zookeeper.java   Source Code and License 5 votes vote down vote up
/**
 * connect ZK, register Watch/unhandle Watch
 * 
 * @return
 */
public CuratorFramework mkClient(Map conf, List<String> servers,
		Object port, String root, final WatcherCallBack watcher) {

	CuratorFramework fk = Utils.newCurator(conf, servers, port, root);

	fk.getCuratorListenable().addListener(new CuratorListener() {
		@Override
		public void eventReceived(CuratorFramework _fk, CuratorEvent e)
				throws Exception {
			if (e.getType().equals(CuratorEventType.WATCHED)) {
				WatchedEvent event = e.getWatchedEvent();

				watcher.execute(event.getState(), event.getType(),
						event.getPath());
			}

		}
	});

	fk.getUnhandledErrorListenable().addListener(
			new UnhandledErrorListener() {
				@Override
				public void unhandledError(String msg, Throwable error) {
					String errmsg = "Unrecoverable Zookeeper error, halting process: "
							+ msg;
					LOG.error(errmsg, error);
					JStormUtils.halt_process(1,
							"Unrecoverable Zookeeper error");

				}
			});
	fk.start();
	return fk;
}
 
Example 52
Project: jstrom   File: SimpleBolt.java   Source Code and License 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context) {
	this.conf = stormConf;

	int timeoutSeconds = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
	counters = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
	
	LOG.info("Successfully do prepare");

}
 
Example 53
Project: learn_jstorm   File: BoltCollector.java   Source Code and License 5 votes vote down vote up
@Override
public void ack(Tuple input) {

	if (ackerNum > 0) {

		Long ack_val = Long.valueOf(0);
		Object pend_val = pending_acks.remove(input);
		if (pend_val != null) {
			ack_val = (Long) (pend_val);
		}

		for (Entry<Long, Long> e : input.getMessageId().getAnchorsToIds()
				.entrySet()) {

			UnanchoredSend.send(
					topologyContext,
					sendTargets,
					taskTransfer,
					Acker.ACKER_ACK_STREAM_ID,
					JStormUtils.mk_list((Object) e.getKey(),
							JStormUtils.bit_xor(e.getValue(), ack_val)));
		}
	}

	Long delta = tuple_time_delta(tuple_start_times, input);
	if (delta != null) {
		task_stats.bolt_acked_tuple(input.getSourceComponent(),
				input.getSourceStreamId(), delta);
	}
}
 
Example 54
Project: learn_jstorm   File: UIUtils.java   Source Code and License 5 votes vote down vote up
public static String getTaskError(List<ErrorInfo> errList) {
	if (errList == null) {
		return "";
	}

	List<String> errors = new ArrayList<String>();

	for (ErrorInfo einfo : errList) {
		long current = System.currentTimeMillis() / 1000;
		
		//shorten the most recent time for "queue is full" error
		int maxTime = JStormUtils.MIN_30;
		if (einfo.get_error().indexOf("queue is full") != -1)
			maxTime = JStormUtils.MIN_1*3;
		else if (einfo.get_error().indexOf("is dead on") != -1)
			maxTime = JStormUtils.DAY_1*3;
		
		if (current - einfo.get_error_time_secs() < maxTime) {
			errors.add(einfo.get_error());
		}
	}

	String rtn = "";
	int size = 0;
	for (String e : errors) {
		if (size >= maxErrornum) {
			break;
		}
		rtn += e + ";";
		size++;
	}
	return rtn;
}
 
Example 55
Project: learn_jstorm   File: WorkerHeartbeatRunable.java   Source Code and License 5 votes vote down vote up
public WorkerHeartbeatRunable(WorkerData workerData) {

		this.workerData = workerData;

		this.conf = workerData.getConf();
		this.worker_id = workerData.getWorkerId();
		this.port = workerData.getPort();
		this.topologyId = workerData.getTopologyId();
		this.task_ids = new CopyOnWriteArraySet<Integer>(
				workerData.getTaskids());
		this.active = workerData.getActive();

		String key = Config.WORKER_HEARTBEAT_FREQUENCY_SECS;
		frequence = JStormUtils.parseInt(conf.get(key), 10);
	}
 
Example 56
Project: jstorm-0.9.6.3-   File: BoltCollector.java   Source Code and License 5 votes vote down vote up
public BoltCollector(int message_timeout_secs, ITaskReportErr report_error,
		TaskSendTargets _send_fn, Map _storm_conf,
		TaskTransfer _transfer_fn, TopologyContext _topology_context,
		Integer task_id, RotatingMap<Tuple, Long> tuple_start_times,
		CommonStatsRolling _task_stats) {

	this.rotateTime = 1000L * message_timeout_secs/(Acker.TIMEOUT_BUCKET_NUM - 1);
	this.reportError = report_error;
	this.sendTargets = _send_fn;
	this.storm_conf = _storm_conf;
	this.taskTransfer = _transfer_fn;
	this.topologyContext = _topology_context;
	this.task_id = task_id;
	this.task_stats = _task_stats;

	this.pending_acks = new RotatingMap<Tuple, Long>(
			Acker.TIMEOUT_BUCKET_NUM);
	// this.pending_acks = new TimeCacheMap<Tuple,
	// Long>(message_timeout_secs,
	// Acker.TIMEOUT_BUCKET_NUM);
	this.tuple_start_times = tuple_start_times;

	this.ackerNum = JStormUtils.parseInt(storm_conf
			.get(Config.TOPOLOGY_ACKER_EXECUTORS));
	
	String componentId = topologyContext.getThisComponentId();
	timer = Metrics.registerTimer(JStormServerUtils.getName(componentId, task_id), 
			MetricDef.EMIT_TIME, String.valueOf(task_id), Metrics.MetricType.TASK);
	random = new Random();
	random.setSeed(System.currentTimeMillis());
}
 
Example 57
Project: jstorm-0.9.6.3-   File: BoltCollector.java   Source Code and License 5 votes vote down vote up
public static void put_xor(RotatingMap<Tuple, Long> pending, Tuple key,
		Long id) {
	// synchronized (pending) {
	Long curr = pending.get(key);
	if (curr == null) {
		curr = Long.valueOf(0);
	}
	pending.put(key, JStormUtils.bit_xor(curr, id));
	// }
}
 
Example 58
Project: learn_jstorm   File: SimpleBolt.java   Source Code and License 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context) {
	this.conf = stormConf;

	int timeoutSeconds = JStormUtils.parseInt(
			conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS), 30);
	counters = new TimeCacheMap<BatchId, AtomicLong>(timeoutSeconds);
	
	LOG.info("Successfully do prepare");

}
 
Example 59
Project: learn_jstorm   File: SpoutCollector.java   Source Code and License 5 votes vote down vote up
public SpoutCollector(Integer task_id, backtype.storm.spout.ISpout spout,
		CommonStatsRolling task_stats, TaskSendTargets sendTargets,
		Map _storm_conf, TaskTransfer _transfer_fn,
		TimeOutMap<Long, TupleInfo> pending,
		TopologyContext topology_context,
		DisruptorQueue disruptorAckerQueue, ITaskReportErr _report_error) {
	this.sendTargets = sendTargets;
	this.storm_conf = _storm_conf;
	this.transfer_fn = _transfer_fn;
	this.pending = pending;
	this.topology_context = topology_context;

	this.disruptorAckerQueue = disruptorAckerQueue;

	this.task_stats = task_stats;
	this.spout = spout;
	this.task_id = task_id;
	this.report_error = _report_error;

	ackerNum = JStormUtils.parseInt(storm_conf
			.get(Config.TOPOLOGY_ACKER_EXECUTORS));
	isDebug = JStormUtils.parseBoolean(
			storm_conf.get(Config.TOPOLOGY_DEBUG), false);

	random = new Random();
	random.setSeed(System.currentTimeMillis());

	String componentId = topology_context.getThisComponentId();
	emitTotalTimer = Metrics.registerTimer(JStormServerUtils.getName(componentId, task_id), 
			MetricDef.EMIT_TIME, String.valueOf(task_id), Metrics.MetricType.TASK);
}
 
Example 60
Project: jstrom   File: NimbusUtils.java   Source Code and License 5 votes vote down vote up
public static void updateTopologyTaskTimeout(NimbusData data, String topologyId) {
    Map topologyConf = null;
    try {
        topologyConf = StormConfig.read_nimbus_topology_conf(data.getConf(), topologyId);
    } catch (IOException e) {
        LOG.warn("Failed to read configuration of " + topologyId + ", " + e.getMessage());
    }

    Integer timeout = JStormUtils.parseInt(topologyConf.get(Config.NIMBUS_TASK_TIMEOUT_SECS));
    if (timeout == null) {
        timeout = JStormUtils.parseInt(data.getConf().get(Config.NIMBUS_TASK_TIMEOUT_SECS));
    }
    LOG.info("Setting taskTimeout:" + timeout + " for " + topologyId);
    data.getTopologyTaskTimeout().put(topologyId, timeout);
}