backtype.storm.Constants Java Examples

The following examples show how to use backtype.storm.Constants. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Common.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static StormTopology add_system_components(StormTopology topology) {
    // generate inputs
    Map<GlobalStreamId, Grouping> inputs = new HashMap<>();

    // generate outputs
    HashMap<String, StreamInfo> outputs = new HashMap<>();
    //ArrayList<String> fields = new ArrayList<String>();

    outputs.put(Constants.SYSTEM_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("rate_secs")));
    outputs.put(Constants.METRICS_TICK_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("interval")));
    outputs.put(Constants.CREDENTIALS_CHANGED_STREAM_ID, Thrift.outputFields(JStormUtils.mk_list("creds")));

    // ComponentCommon common = new ComponentCommon(inputs, outputs);

    IBolt ackerbolt = new SystemBolt();
    Bolt bolt = Thrift.mkBolt(inputs, ackerbolt, outputs, 0);
    topology.put_to_bolts(Constants.SYSTEM_COMPONENT_ID, bolt);

    add_system_streams(topology);

    return topology;

}
 
Example #2
Source File: CoordinatedBolt.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public void prepare(Map config, TopologyContext context, OutputCollector collector) {
    TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null;
    if (_delegate instanceof TimeoutCallback) {
        callback = new TimeoutItems();
    }
    _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback);
    _collector = collector;
    _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector)));
    for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()).keySet()) {
        for (Integer task : context.getComponentTasks(component)) {
            _countOutTasks.add(task);
        }
    }
    if (!_sourceArgs.isEmpty()) {
        _numSourceReports = 0;
        for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) {
            if (entry.getValue().singleCount) {
                _numSourceReports += 1;
            } else {
                _numSourceReports += context.getComponentTasks(entry.getKey()).size();
            }
        }
    }
}
 
Example #3
Source File: ShellBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private BoltMsg createHeartbeatBoltMessage(String genId) {
    BoltMsg msg = new BoltMsg();
    msg.setId(genId);
    msg.setTask(Constants.SYSTEM_TASK_ID);
    msg.setStream(HEARTBEAT_STREAM_ID);
    msg.setTuple(new ArrayList<>());
    return msg;
}
 
Example #4
Source File: GeneralTopologyContext.java    From jstorm with Apache License 2.0 5 votes vote down vote up
/**
 * Gets the component id for the specified task id.
 * The component id maps to a component id specified for a Spout or Bolt in the topology definition.
 *
 * @param taskId the task id
 * @return the component id for the input task id
 */
public String getComponentId(int taskId) {
    if (taskId == Constants.SYSTEM_TASK_ID) {
        return Constants.SYSTEM_COMPONENT_ID;
    } else {
        return _taskToComponent.get(taskId);
    }
}
 
Example #5
Source File: CoordinatedBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private TupleType getTupleType(Tuple tuple) {
    if (_idStreamSpec != null && tuple.getSourceGlobalStreamid().equals(_idStreamSpec._id)) {
        return TupleType.ID;
    } else if (!_sourceArgs.isEmpty() && tuple.getSourceStreamId().equals(Constants.COORDINATED_STREAM_ID)) {
        return TupleType.COORD;
    } else {
        return TupleType.REGULAR;
    }
}
 
Example #6
Source File: Httpserver.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public LogHandler(Map conf) {
    this.pageSize = ConfigExtension.getLogPageSize(conf);
    logDir = JStormUtils.getLogDir();
    String logDirPath = PathUtils.getCanonicalPath(logDir);
    if (logDirPath == null) {
        accessDirs.add(logDir);
    } else {
        accessDirs.add(logDirPath);
    }

    stormHome = System.getProperty("jstorm.home");
    if (stormHome != null) {
        String stormHomePath = PathUtils.getCanonicalPath(stormHome);
        if (stormHomePath == null) {
            accessDirs.add(stormHome);
        } else {
            accessDirs.add(stormHomePath);
        }
    }

    String confDir = System.getProperty(Constants.JSTORM_CONF_DIR);
    if (!StringUtils.isBlank(confDir)) {
        String confDirPath = PathUtils.getCanonicalPath(confDir);
        if (confDirPath != null) {
            accessDirs.add(confDirPath);
        }
    }

    this.conf = conf;
    LOG.info("logview log dir=" + logDir);
}
 
Example #7
Source File: BoltExecutors.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public BoltExecutors(Task task) {
    super(task);

    this.bolt = (IBolt) task.getTaskObj();

    // create TimeCacheMap
    this.tupleStartTimes = new RotatingMap<>(Acker.TIMEOUT_BUCKET_NUM);
    this.ackerNum = JStormUtils.parseInt(storm_conf.get(Config.TOPOLOGY_ACKER_EXECUTORS));

    // create BoltCollector
    BoltCollector outputCollector;
    if (ConfigExtension.isTaskBatchTuple(storm_conf)) {
        outputCollector = new BoltBatchCollector(task, tupleStartTimes, messageTimeoutSecs);
    } else {
        outputCollector = new BoltCollector(task, tupleStartTimes, messageTimeoutSecs);
    }
    this.outputCollector = new OutputCollector(outputCollector);

    //this task don't continue until it bulid connection with topologyMaster
    Integer topologyId = sysTopologyCtx.getTopologyMasterId();
    List<Integer> localWorkerTasks = sysTopologyCtx.getThisWorkerTasks();
    if (topologyId != 0 && !localWorkerTasks.contains(topologyId)) {
        while (getConnection(topologyId) == null) {
            JStormUtils.sleepMs(10);
            LOG.info("this task still is building connection with topology Master");
        }
    }

    taskHbTrigger.setBoltOutputCollector(outputCollector);
    taskHbTrigger.register();
    Object tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_MS);
    if (tickFrequency == null) {
        tickFrequency = storm_conf.get(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS);
        if (tickFrequency != null)
            tickFrequency = JStormUtils.parseInt(tickFrequency) * 1000;
    }

    isSystemBolt = Common.isSystemComponent(componentId);
    if (tickFrequency != null && !isSystemBolt) {
        Integer frequency = JStormUtils.parseInt(tickFrequency);
        tickTupleTrigger = new TickTupleTrigger(
                sysTopologyCtx, frequency, idStr + Constants.SYSTEM_TICK_STREAM_ID, controlQueue);
        tickTupleTrigger.register();
    }

    LOG.info("Successfully create BoltExecutors " + idStr);
}
 
Example #8
Source File: TupleHelpers.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public static boolean isTickTuple(Tuple tuple) {
    return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) &&
            tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #9
Source File: TupleUtils.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public static boolean isTick(Tuple tuple) {
    return tuple != null && Constants.SYSTEM_COMPONENT_ID.equals(tuple.getSourceComponent())
            && Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
}
 
Example #10
Source File: CoordinatedBolt.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    _delegate.declareOutputFields(declarer);
    declarer.declareStream(Constants.COORDINATED_STREAM_ID, true, new Fields("id", "count"));
}
 
Example #11
Source File: LinearDRPCTopologyBuilder.java    From jstorm with Apache License 2.0 4 votes vote down vote up
private StormTopology createTopology(DRPCSpout spout) {
    final String SPOUT_ID = "spout";
    final String PREPARE_ID = "prepare-request";

    TopologyBuilder builder = new TopologyBuilder();
    builder.setSpout(SPOUT_ID, spout);
    builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID);
    int i = 0;
    for (; i < _components.size(); i++) {
        Component component = _components.get(i);

        Map<String, SourceArgs> source = new HashMap<String, SourceArgs>();
        if (i == 1) {
            source.put(boltId(i - 1), SourceArgs.single());
        } else if (i >= 2) {
            source.put(boltId(i - 1), SourceArgs.all());
        }
        IdStreamSpec idSpec = null;
        if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) {
            idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM);
        }
        BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism);

        for (Map conf : component.componentConfs) {
            declarer.addConfigurations(conf);
        }

        if (idSpec != null) {
            declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request"));
        }
        if (i == 0 && component.declarations.isEmpty()) {
            declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM);
        } else {
            String prevId;
            if (i == 0) {
                prevId = PREPARE_ID;
            } else {
                prevId = boltId(i - 1);
            }
            for (InputDeclaration declaration : component.declarations) {
                declaration.declare(prevId, declarer);
            }
        }
        if (i > 0) {
            declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID);
        }
    }

    IRichBolt lastBolt = _components.get(_components.size() - 1).bolt;
    OutputFieldsGetter getter = new OutputFieldsGetter();
    lastBolt.declareOutputFields(getter);
    Map<String, StreamInfo> streams = getter.getFieldsDeclaration();
    if (streams.size() != 1) {
        throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology");
    }
    String outputStream = streams.keySet().iterator().next();
    List<String> fields = streams.get(outputStream).get_output_fields();
    if (fields.size() != 2) {
        throw new RuntimeException(
                "Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result.");
    }

    builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0)))
            .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request"));
    i++;
    builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult");
    return builder.createTopology();
}
 
Example #12
Source File: SpringBolt.java    From storm-solr with Apache License 2.0 4 votes vote down vote up
public static final boolean isTickTuple(final Tuple tuple) {
  return tuple != null &&
    Constants.SYSTEM_COMPONENT_ID.equals(tuple.getSourceComponent()) &&
    Constants.SYSTEM_TICK_STREAM_ID.equals(tuple.getSourceStreamId());
}
 
Example #13
Source File: TickTupleTrigger.java    From jstorm with Apache License 2.0 4 votes vote down vote up
@Override
public void updateObject() {
    this.object = new TupleImplExt(topologyContext,
            new Values(TimeUtils.current_time_secs()), (int) Constants.SYSTEM_TASK_ID, Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #14
Source File: TupleHelpers.java    From jstorm with Apache License 2.0 4 votes vote down vote up
public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals(
      Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #15
Source File: Tuples.java    From monasca-common with Apache License 2.0 4 votes vote down vote up
public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)
      && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #16
Source File: MockTupleHelpers.java    From storm-benchmark with Apache License 2.0 4 votes vote down vote up
public static Tuple mockTickTuple() {
  return mockTuple(Constants.SYSTEM_COMPONENT_ID, Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #17
Source File: TupleHelpers.java    From storm-benchmark with Apache License 2.0 4 votes vote down vote up
public static boolean isTickTuple(Tuple tuple) {
  return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID) && tuple.getSourceStreamId().equals(
      Constants.SYSTEM_TICK_STREAM_ID);
}
 
Example #18
Source File: RichTickTupleBolt.java    From storm-trident-elasticsearch with Apache License 2.0 4 votes vote down vote up
private static boolean isTickTuple(Tuple tuple) {
    return tuple.getSourceComponent().equals(Constants.SYSTEM_COMPONENT_ID)
            && tuple.getSourceStreamId().equals(Constants.SYSTEM_TICK_STREAM_ID);
}