backtype.storm.task.IBolt Java Examples

public static StormTopology add_system_components(StormTopology topology) {
    // generate inputs
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();

    // generate outputs
    HashMap<String, StreamInfo> outputs = new HashMap<>();
    //ArrayList<String> fields = new ArrayList<String>();

    outputs.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("rate_secs")));
    outputs.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("interval")));
    outputs.put(Constants.CREDENTIALS_CHANGED_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("creds")));

    // ComponentCommon common = new ComponentCommon(inputs, outputs);

    IBolt ackerbolt = new SystemBolt();
    Bolt bolt = Thrift.mkBolt(inputs, ackerbolt, outputs, 0);
    topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, bolt);

    add_system_streams(topology);

    return topology;

}
 

public BaseExecutors mkExecutor() {
    BaseExecutors baseExecutor = null;

    if (taskObj instanceof IBolt) {
        if (taskId == topologyContext.getTopologyMasterId())
            baseExecutor = new TopologyMasterBoltExecutors(this);
        else
            baseExecutor = new BoltExecutors(this);
    } else if (taskObj instanceof ISpout) {
        if (isSingleThread(stormConf)) {
            baseExecutor = new SingleThreadSpoutExecutors(this);
        } else {
            baseExecutor = new MultipleThreadSpoutExecutors(this);
        }
    }

    return baseExecutor;
}
 

public static Bolt mkBolt(Map<GlobalStreamId, Grouping> inputs, IBolt bolt,
                          HashMap<String, StreamInfo> output, Integer p) {
    ComponentCommon common = mkComponentcommon(inputs, output, p);
    byte[] boltSer = Utils.serialize(bolt);
    ComponentObject component = ComponentObject.serialized_java(boltSer);
    return new Bolt(component, common);
}
 

private void closeComponent(Object _task_obj) {
    if (_task_obj instanceof IBolt) {
        ((IBolt) _task_obj).cleanup();
    }

    if (_task_obj instanceof ISpout) {
        ((ISpout) _task_obj).close();
    }
}
 

@SuppressWarnings("unchecked")
@Override
public void prepare(final Map stormConf, final TopologyContext context, final OutputCollector collector) {
    IFn hof = Utils.loadClojureFn(_fnSpec.get(0), _fnSpec.get(1));
    try {
        IFn preparer = (IFn) hof.applyTo(RT.seq(_params));
        final Map<Keyword, Object> collectorMap =
                new PersistentArrayMap(new Object[]{Keyword.intern(Symbol.create("output-collector")), collector,
                        Keyword.intern(Symbol.create("context")), context});
        List<Object> args = new ArrayList<Object>() {
            {
                add(stormConf);
                add(context);
                add(collectorMap);
            }
        };

        _bolt = (IBolt) preparer.applyTo(RT.seq(args));
        // this is kind of unnecessary for clojure
        try {
            _bolt.prepare(stormConf, context, collector);
        } catch (AbstractMethodError ignored) {
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

public TaskReceiver(Task task, int taskId, Map stormConf, TopologyContext topologyContext,
                    Map<Integer, DisruptorQueue> innerTaskTransfer,
                    TaskStatus taskStatus, String taskName) {
    this.stormConf = stormConf;
    this.task = task;
    this.bolt = (task.getTaskObj() instanceof IBolt ? (IBolt) task.getTaskObj() : null);
    this.taskId = taskId;
    this.idStr = taskName;

    this.topologyContext = topologyContext;
    this.innerTaskTransfer = innerTaskTransfer;

    this.taskStatus = taskStatus;

    int queueSize = JStormUtils.parseInt(stormConf.get(Config.TOPOLOGY_EXECUTOR_RECEIVE_BUFFER_SIZE), 256);

    long timeout = JStormUtils.parseLong(stormConf.get(Config.TOPOLOGY_DISRUPTOR_WAIT_TIMEOUT), 10);
    WaitStrategy waitStrategy = new TimeoutBlockingWaitStrategy(timeout, TimeUnit.MILLISECONDS);
    boolean isDisruptorBatchMode = ConfigExtension.isDisruptorQueueBatchMode(stormConf);
    int disruptorBatch = ConfigExtension.getDisruptorBufferSize(stormConf);
    long flushMs = ConfigExtension.getDisruptorBufferFlushMs(stormConf);
    this.deserializeQueue = DisruptorQueue.mkInstance("TaskDeserialize", ProducerType.MULTI, queueSize, waitStrategy,
            isDisruptorBatchMode, disruptorBatch, flushMs);
    dserializeThreadNum = ConfigExtension.getTaskDeserializeThreadNum(stormConf);
    deserializeThreads = new ArrayList<>();
    setDeserializeThread();
    //this.deserializer = new KryoTupleDeserializer(stormConf, topologyContext);

    String topologyId = topologyContext.getTopologyId();
    String component = topologyContext.getThisComponentId();

    AsmHistogram deserializeTimerHistogram = new AsmHistogram();
    deserializeTimerHistogram.setAggregate(false);
    deserializeTimer = (AsmHistogram) JStormMetrics.registerTaskMetric(MetricUtils.taskMetricName(
            topologyId, component, taskId, MetricDef.DESERIALIZE_TIME, MetricType.HISTOGRAM), deserializeTimerHistogram);

    QueueGauge deserializeQueueGauge = new QueueGauge(deserializeQueue, idStr, MetricDef.DESERIALIZE_QUEUE);
    JStormMetrics.registerTaskMetric(MetricUtils.taskMetricName(
                    topologyId, component, taskId, MetricDef.DESERIALIZE_QUEUE, MetricType.GAUGE),
            new AsmGauge(deserializeQueueGauge));
    JStormHealthCheck.registerTaskHealthCheck(taskId, MetricDef.DESERIALIZE_QUEUE, deserializeQueueGauge);

    this.isBackpressureEnable = ConfigExtension.isBackpressureEnable(stormConf);
    this.highMark = (float) ConfigExtension.getBackpressureWaterMarkHigh(stormConf);
    this.lowMark = (float) ConfigExtension.getBackpressureWaterMarkLow(stormConf);
    this.backpressureStatus = false;

    LOG.info("Successfully started TaskReceiver thread for {}, thread num: {}", idStr, dserializeThreadNum);
}
 

public BoltExecutors(Task task) {
    super(task);

    this.bolt = (IBolt) task.getTaskObj();

    // create TimeCacheMap
    this.tupleStartTimes = new RotatingMap<>(Acker.TIMEOUT_BUCKET_NUM);
    this.ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));

    // create BoltCollector
    BoltCollector outputCollector;
    if (ConfigExtension.isTaskBatchTuple(storm_conf)) {
        outputCollector = new BoltBatchCollector(task, tupleStartTimes, messageTimeoutSecs);
    } else {
        outputCollector = new BoltCollector(task, tupleStartTimes, messageTimeoutSecs);
    }
    this.outputCollector = new OutputCollector(outputCollector);

    //this task don't continue until it bulid connection with topologyMaster
    Integer topologyId = sysTopologyCtx.getTopologyMasterId();
    List<Integer> localWorkerTasks = sysTopologyCtx.getThisWorkerTasks();
    if (topologyId != 0 && !localWorkerTasks.contains(topologyId)) {
        while (getConnection(topologyId) == null) {
            JStormUtils.sleepMs(10);
            LOG.info("this task still is building connection with topology Master");
        }
    }

    taskHbTrigger.setBoltOutputCollector(outputCollector);
    taskHbTrigger.register();
    Object tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_MS);
    if (tickFrequency == null) {
        tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
        if (tickFrequency != null)
            tickFrequency = JStormUtils.parseInt(tickFrequency) * 1000;
    }

    isSystemBolt = Common.isSystemComponent(componentId);
    if (tickFrequency != null && !isSystemBolt) {
        Integer frequency = JStormUtils.parseInt(tickFrequency);
        tickTupleTrigger = new TickTupleTrigger(
                sysTopologyCtx, frequency, idStr + Constants.SYSTEM_TICK_STREAM_ID, controlQueue);
        tickTupleTrigger.register();
    }

    LOG.info("Successfully create BoltExecutors " + idStr);
}
 

public TaskBundle(IBolt task, int componentId) {
    this.task = task;
    this.componentId = componentId;
}
 

public NonRichBoltTracker(IBolt delegate, String id) {
    _delegate = delegate;
    _trackId = id;
}