Java Code Examples for backtype.storm.task.TopologyContext#getThisTaskId()

The following examples show how to use backtype.storm.task.TopologyContext#getThisTaskId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: CoordinatedBolt.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {

        taskId = String.valueOf(context.getThisTaskId());
        taskName = context.getThisComponentId() + "_" + context.getThisTaskId();

        this.basicCollector = new BasicOutputCollector(collector);
        this.collector = collector;

        if (delegate instanceof ICommitter) {
            isCommiter = true;
            commited = new TimeCacheMap<>(context.maxTopologyMessageTimeout());
            mkCommitDir(conf);
        }

        delegate.prepare(conf, context);

    }
 
Example 2
Source File: BatchCache.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public BatchCache(TopologyContext context, StormTopology sysTopology, boolean isIntraWorker) {
    this.context = context;
    this.stormConf = context.getStormConf();

    this.isExactlyOnceMode = JStormUtils.parseBoolean(stormConf.get("transaction.exactly.once.mode"), true);

    this.pendingBatches = new ConcurrentHashMap<>();

    serializer = new KryoTupleSerializer(stormConf, sysTopology);
    deserializer = new KryoTupleDeserializer(stormConf, context, sysTopology);

    String cacheDir = context.getWorkerIdDir() + "/transactionCache/task-" + context.getThisTaskId();
    if (isIntraWorker)
        cacheDir += "/intra";
    else
        cacheDir += "/inter";
    String cacheType = Utils.getString(stormConf.get("transaction.exactly.cache.type"), "default");
    if (cacheType.equalsIgnoreCase("rocksDb")) {
        cacheOperator = new RocksDbCacheOperator(context, cacheDir);
    } else {
        cacheOperator = new DefaultCacheOperator();
    }
    LOG.info("Cache config: isExactlyOnce={}, cacheType={}", isExactlyOnceMode, cacheType);
}
 
Example 3
Source File: TopologyMasterContext.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public TopologyMasterContext(Map stormConf, TopologyContext context,
                             final OutputCollector collector) {
    this.conf = context.getStormConf();
    this.context = context;
    this.collector = collector;
    this.taskId = context.getThisTaskId();
    this.topologyId = context.getTopologyId();
    this.zkCluster = context.getZkCluster();

    workerSet = new AtomicReference<>();
    try {
        Assignment assignment = zkCluster.assignment_info(topologyId, null);
        this.workerSet.set(assignment.getWorkers());
    } catch (Exception e) {
        LOG.error("Failed to get assignment for " + topologyId);
        throw new RuntimeException(e);
    }

    this.topologyMetricContext = new TopologyMetricContext(topologyId, workerSet.get(), conf);
}
 
Example 4
Source File: JobHistorySpout.java    From eagle with Apache License 2.0 5 votes vote down vote up
private int calculatePartitionId(TopologyContext context) {
    int thisGlobalTaskId = context.getThisTaskId();
    String componentName = context.getComponentId(thisGlobalTaskId);
    List<Integer> globalTaskIds = context.getComponentTasks(componentName);
    numTotalPartitions = globalTaskIds.size();
    int index = 0;
    for (Integer id : globalTaskIds) {
        if (id == thisGlobalTaskId) {
            return index;
        }
        index++;
    }
    throw new IllegalStateException();
}
 
Example 5
Source File: AckTransactionBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.batchXorTracker = new AckPendingBatchTracker<>();
    this.ackOutputCollector = new AckOutputCollector(collector.getDelegate(), batchXorTracker);
    this.bolt.prepare(stormConf, context, new OutputCollector(ackOutputCollector));

    this.componentId = context.getThisComponentId();
    this.taskId = context.getThisTaskId();
    this.taskStats = new TaskBaseMetric(context.getTopologyId(), componentId, taskId);
    this.batchTimeout = ConfigExtension.getTransactionBatchSnapshotTimeout(stormConf) * 1000;
    this.random = new Random(Utils.secureRandomLong());

    LOG.info("batchTimeout: {}", batchTimeout);
}
 
Example 6
Source File: AckTransactionSpout.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    SpoutOutputCollectorCb ackOutput = new AckSpoutOutputCollector(collector.getDelegate());
    spoutExecutor.open(conf, context, new SpoutOutputCollector(ackOutput));
    tracker = new AckPendingBatchTracker<>();
    taskStats = new TaskBaseMetric(context.getTopologyId(), context.getThisComponentId(), context.getThisTaskId());
    random = new Random(Utils.secureRandomLong());
}
 
Example 7
Source File: SimpleFileNameFormat.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public void prepare(Map conf, TopologyContext topologyContext) {
    this.componentId = topologyContext.getThisComponentId();
    this.taskId = topologyContext.getThisTaskId();
    try {
        this.host = InetAddress.getLocalHost().getCanonicalHostName();
    } catch (UnknownHostException e) {
        throw new RuntimeException(e);
    }
}
 
Example 8
Source File: SequenceSpout.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    
    if (conf.get("spout.max.sending.num") == null) {
        isLimited = false;
    } else {
        isLimited = true;
        SPOUT_MAX_SEND_NUM = JStormUtils.parseLong(conf.get("spout.max.sending.num"));
    }
    
    Boolean btrue = JStormUtils.parseBoolean(conf.get("spout.send.contrl.message"));
    if (btrue != null && btrue) {
        isSendCtrlMsg = true;
    } else {
        isSendCtrlMsg = false;
    }
    
    isFinished = false;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    MetricClient metricClient = new MetricClient(context);
    this.tpCounter = metricClient.registerTopologyCounter("TpCounter");

    MAX_PENDING_COUNTER = getMaxPending(conf);
    
    bufferLen = JStormUtils.parseInt(conf.get("byte.buffer.len"), 0);
    
    random = new Random();
    random.setSeed(System.currentTimeMillis());

    idGenerate = new Random(Utils.secureRandomLong());

    JStormUtils.sleepMs(20 * 1000);
    
    LOG.info("Finish open, buffer Len:" + bufferLen);
}
 
Example 9
Source File: MergeRecord.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    
    LOG.info("Finished preparation");
}
 
Example 10
Source File: TotalCount.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    this.context = context;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());

    checkTupleId = JStormUtils.parseBoolean(stormConf.get("bolt.check.tupleId"), false);
    slowDonw = JStormUtils.parseBoolean(stormConf.get("bolt.slow.down"), false);
    
    metricClient = new MetricClient(context);
    
    Gauge<Double> gauge = new Gauge<Double>() {
        private Random random = new Random();
        
        @Override
        public Double getValue() {
            return random.nextDouble();
        }
        
    };
    myGauge = metricClient.registerGauge("myGauge", gauge);
    myCounter = metricClient.registerCounter("myCounter");
    tpCounter = metricClient.registerTopologyCounter("TpCounter");
    myMeter = metricClient.registerMeter("myMeter");
    myJStormHistogram = metricClient.registerHistogram("myHistogram");
    
    LOG.info("Finished preparation " + stormConf);
}
 
Example 11
Source File: Latency.java    From flink-perf with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
	this.spoutOutputCollector = spoutOutputCollector;
	this.tid = topologyContext.getThisTaskId();
	try {
		this.host = InetAddress.getLocalHost().getHostName();
	} catch (UnknownHostException e) {
		e.printStackTrace();
	}
}
 
Example 12
Source File: HadoopLogAccumulatorBolt.java    From eagle with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.taskId = context.getThisTaskId();
    this.collector = collector;
    this.client = new HadoopLogTrafficPersist(config);
    this.accumulator = new SimpleWindowCounter(windowSize);
}
 
Example 13
Source File: JobRunningSpout.java    From Eagle with Apache License 2.0 5 votes vote down vote up
/**
 * TODO: just copy this part from jobHistorySpout, need to move it to a common place
 * @param context
 * @return
 */
private int calculatePartitionId(TopologyContext context){
	int thisGlobalTaskId = context.getThisTaskId();
	String componentName = context.getComponentId(thisGlobalTaskId);
	List<Integer> globalTaskIds = context.getComponentTasks(componentName);
	int index = 0;
	for(Integer id : globalTaskIds){
		if(id == thisGlobalTaskId){
			return index;
		}
		index++;
	}
	throw new IllegalStateException();
}
 
Example 14
Source File: PairCount.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public void prepare(Map conf, TopologyContext context) {
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    LOG.info("Successfully do parepare " + context.getThisComponentId());
}
 
Example 15
Source File: TMUdfSpout.java    From jstorm with Apache License 2.0 4 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.topologyMasterId = context.getTopologyMasterId();
    this.spoutTaskId = context.getThisTaskId();
    this.collector = collector;
}
 
Example 16
Source File: TestEventLogSpout.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    this.source = context.getThisTaskId();
    long taskCount = context.getComponentTasks(context.getThisComponentId()).size();
    myCount = totalCount / taskCount;
}
 
Example 17
Source File: DefaultFileNameFormat.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public void prepare(Map conf, TopologyContext topologyContext) {
    this.componentId = topologyContext.getThisComponentId();
    this.taskId = topologyContext.getThisTaskId();
}
 
Example 18
Source File: DefaultFileNameFormat.java    From storm-hdfs with Apache License 2.0 4 votes vote down vote up
@Override
public void prepare(Map conf, TopologyContext topologyContext) {
    this.componentId = topologyContext.getThisComponentId();
    this.taskId = topologyContext.getThisTaskId();
}
 
Example 19
Source File: MetricClient.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public MetricClient(TopologyContext context) {
    taskId = context.getThisTaskId();
    this.topologyId = context.getTopologyId();
    this.componentId = context.getThisComponentId();
}
 
Example 20
Source File: StatefulWindowedBoltExecutor.java    From jstorm with Apache License 2.0 4 votes vote down vote up
private KeyValueState<TaskStream, WindowState> getWindowState(Map stormConf, TopologyContext context) {
    String namespace = context.getThisComponentId() + "-" + context.getThisTaskId() + "-window";
    return (KeyValueState<TaskStream, WindowState>) StateFactory.getState(namespace, stormConf, context);
}