backtype.storm.task.TopologyContext Java Examples

The following examples show how to use backtype.storm.task.TopologyContext. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BatchInputBolt.java    From StormCV with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
void prepare(Map conf, TopologyContext context) {
	// use TTL and maxSize from config if they were not set explicitly using the constructor (implicit way of doing this...)
	if(TTL == 29) TTL = conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC) == null ? TTL : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC)).intValue();
	if(maxSize == 256) maxSize = conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE) == null ? maxSize : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE)).intValue();
	history = new History(this);
	
	// IF NO grouping was set THEN select the first grouping registered for the spout as the grouping used within the Spout (usually a good guess)
	if(groupBy == null){
		Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
		for(GlobalStreamId id : sources.keySet()){
			Grouping grouping = sources.get(id);
			this.groupBy = new Fields(grouping.get_fields());
			break;
		}
	}
	
	// prepare the selector and operation
	try {
		batcher.prepare(conf);
		operation.prepare(conf, context);
	} catch (Exception e) {
		logger.error("Unable to preapre the Selector or Operation", e);
	}
}
 
Example #2
Source File: TaskHeartbeatTrigger.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public TaskHeartbeatTrigger(Map conf, String name, DisruptorQueue controlQueue, int taskId, String componentId,
                            TopologyContext sysTopologyCtx, ITaskReportErr reportError, TaskStatus executorStatus) {
    this.name = name;
    this.queue = controlQueue;
    this.opCode = TimerConstants.TASK_HEARTBEAT;

    this.taskId = taskId;
    this.componentId = componentId;
    this.sysTopologyCtx = sysTopologyCtx;

    this.frequency = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
    this.firstTime = 0;

    this.executeThreadHbTime = TimeUtils.current_time_secs();
    this.taskHbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_TIMEOUT_SECS), 180);
    this.intervalCheck = new IntervalCheck();
    this.intervalCheck.setInterval(taskHbTimeout);
    this.intervalCheck.start();

    this.reportError = reportError;

    this.uptime = new UptimeComputer();
    this.executorStatus = executorStatus;
}
 
Example #3
Source File: SVMBolt.java    From senti-storm with Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(Map config, TopologyContext context) {
  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  LOG.info("Loading SVM model...");
  m_dataset = Configuration.getDataSetSemEval2013();
  m_model = SerializationUtils.deserialize(m_dataset.getDatasetPath()
      + File.separator + SVM.SVM_MODEL_FILE_SER);

  if (m_model == null) {
    LOG.error("Could not load SVM model! File: " + m_dataset.getDatasetPath()
        + File.separator + SVM.SVM_MODEL_FILE_SER);
    throw new RuntimeException();
  }
}
 
Example #4
Source File: SystemCheckPersistBolt.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
    this.collector = outputCollector;
    this.client = new EagleServiceClientImpl(config.getConfig());
    if (this.batchSize > 0) {
        this.batchSender = client.batch(this.batchSize);
    }
    this.rackResolver = new DefaultTopologyRackResolver();
    if (config.dataExtractorConfig.resolverCls != null) {
        try {
            rackResolver = config.dataExtractorConfig.resolverCls.newInstance();
            rackResolver.prepare(config);
        } catch (InstantiationException | IllegalAccessException e) {
            LOG.error(e.getMessage(), e);
        }
    }
}
 
Example #5
Source File: ForwardingMetricsConsumer.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
    String[] parts = ((String) registrationArgument).split(":", 2);
    host = parts[0];
    port = Integer.valueOf(parts[1]);
    try {
        socket = new Socket(host, port);
        out = socket.getOutputStream();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example #6
Source File: FetchAndOperateFetcher.java    From StormCV with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	fetcher.prepare(stormConf, context);
	if(sio) singleInOp.prepare(stormConf, context);
	else batchOp.prepare(stormConf, context);
	results = new ArrayList<CVParticle>();
}
 
Example #7
Source File: IRichSpoutDelegate.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Override
public void update(org.apache.heron.api.topology.TopologyContext topologyContext) {
  if (delegate instanceof IUpdatable) {
    ((IUpdatable) delegate).update(topologyContext);
  } else {
    LOG.warning(String.format("Update() event received but can not call update() on delegate "
        + "because it does not implement %s: %s", IUpdatable.class.getName(), delegate));
  }
}
 
Example #8
Source File: CustomStreamGroupingDelegate.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(org.apache.heron.api.topology.TopologyContext context,
                    String component, String streamId,
                    List<Integer> targetTasks) {
  TopologyContext c = new TopologyContext(context);
  GlobalStreamId g = new GlobalStreamId(component, streamId);
  delegate.prepare(c, g, targetTasks);
}
 
Example #9
Source File: FastWordCountIngestionTimeWindowTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    startTime = System.currentTimeMillis();
    maxSendNum = JStormUtils.parseLong(conf.get("max.send.num"), 1000L);
}
 
Example #10
Source File: SequenceFileBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 
Example #11
Source File: CsvHdfsBolt.java    From cognition with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context) {
  try {
    prepareHDFS();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #12
Source File: TestBackpressure.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.conf = conf;
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = JStormUtils.parseInt(conf.get("send.num.each.time"), 1);
    isStatEnable = JStormUtils.parseBoolean(conf.get("is.stat.enable"), true);
}
 
Example #13
Source File: MjpegStreamingOp.java    From StormCV with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	images = MjpegStreamingOp.getImages();
	
	ApplicationAdapter connector = new ApplicationAdapter(new MjpegStreamingOp());
	server = HttpServerFactory.create ("http://localhost:"+port+"/", connector);
	server.start();		
}
 
Example #14
Source File: DRPCSpout.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _clients = new ArrayList<>();
    if (_local_drpc_id == null) {
        _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>());
        _futures = new LinkedList<>();

        int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
        int index = context.getThisTaskIndex();

        int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
        List<String> servers = NetWorkUtils.host2Ip((List<String>) conf.get(Config.DRPC_SERVERS));

        if (servers == null || servers.isEmpty()) {
            throw new RuntimeException("No DRPC servers configured for topology");
        }

        if (numTasks < servers.size()) {
            for (String s : servers) {
                _futures.add(_backround.submit(new Adder(s, port, conf)));
            }
        } else {
            int i = index % servers.size();
            _futures.add(_backround.submit(new Adder(servers.get(i), port, conf)));
        }
    }

}
 
Example #15
Source File: TestHiveJobFetchSpout.java    From eagle with Apache License 2.0 5 votes vote down vote up
@Test
public void testOpen() throws Exception {
    Map conf = mock(HashMap.class);
    TopologyContext context = mock(TopologyContext.class);
    SpoutOutputCollector collector = mock(SpoutOutputCollector.class);
    when(context.getThisTaskId()).thenReturn(1);
    when(context.getComponentTasks(anyString())).thenReturn(new ArrayList<Integer>() {{
        add(1);
    }});
    HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
    HiveJobFetchSpout spout = (HiveJobFetchSpout)provider.getSpout(config, 1);
    spout.open(conf, context, collector);
    Long yesterday = Long.valueOf(new String(curator.getData().forPath(SHARE_RESOURCES)));
    Assert.assertTrue(System.currentTimeMillis() > yesterday);
}
 
Example #16
Source File: KafkaSpout.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
	this.collector = collector;
	if (this.config == null) {
		config = new KafkaSpoutConfig();
	}
	config.configure(conf);
	zkState = new ZkState(conf, config);
	coordinator = new PartitionCoordinator(conf, config, context, zkState);
	lastUpdateMs = System.currentTimeMillis();
}
 
Example #17
Source File: JStormUtils.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public static void reportError(TopologyContext topologyContext, String errorMessge) {
    StormClusterState zkCluster = topologyContext.getZkCluster();
    String topologyId = topologyContext.getTopologyId();
    int taskId = topologyContext.getThisTaskId();

    try {
        zkCluster.report_task_error(topologyId, taskId, errorMessge);
    } catch (Exception e) {
        LOG.warn("Failed to report Error");
    }
}
 
Example #18
Source File: FeatureGenerationBolt.java    From senti-storm with Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map config, TopologyContext context) {
  this.m_dataset = Configuration.getDataSetSemEval2013();

  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  // TODO serialize CombinedFeatureVectorGenerator
  List<FeaturedTweet> featuredTrainTweets = SerializationUtils
      .deserialize(m_dataset.getTrainDataSerializationFile());
  if (featuredTrainTweets != null) {
    TweetTfIdf tweetTfIdf = TweetTfIdf.createFromTaggedTokens(
        FeaturedTweet.getTaggedTokensFromTweets(featuredTrainTweets),
        TfType.LOG, TfIdfNormalization.COS, true);

    LOG.info("Load CombinedFeatureVectorGenerator...");
    m_fvg = new CombinedFeatureVectorGenerator(true, tweetTfIdf);

  } else {
    LOG.error("TaggedTweets could not be found! File is missing: "
        + m_dataset.getTrainDataSerializationFile());
  }
}
 
Example #19
Source File: WindowedBoltExecutor.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private Set<GlobalStreamId> getComponentStreams(TopologyContext context) {
    Set<GlobalStreamId> streams = new HashSet<>();
    for (GlobalStreamId streamId : context.getThisSources().keySet()) {
        if (!streamId.get_streamId().equals(CheckpointSpout.CHECKPOINT_STREAM_ID)) {
            streams.add(streamId);
        }
    }
    return streams;
}
 
Example #20
Source File: RocksDbKvStoreManagerFactory.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void initMonitor(TopologyContext context) {
    isEnableMetrics = true;
    metricClient = new MetricClient(context);
    rocksDbFlushAndCpLatency = metricClient.registerHistogram("RocksDB flush and checkpoint latency");
    hdfsWriteLatency = metricClient.registerHistogram("HDFS write latency");
    hdfsDeleteLatency = metricClient.registerHistogram("HDFS delete latency");
}
 
Example #21
Source File: RollingCountBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    lastModifiedTracker = new NthLastModifiedTimeTracker(
            deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds));
}
 
Example #22
Source File: PerformanceTestTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    this.count = 0l;
    this.isAckRequired = Utils.getBoolean(conf.get("is.ack.required"), false);
    this.sendNumEachTime = Utils.getInt(conf.get("send.num.each.time"), 1);
    this.startTimeStamp = System.currentTimeMillis();
}
 
Example #23
Source File: Common.java    From jstorm with Apache License 2.0 5 votes vote down vote up
/**
 * get current task's output <Stream_id, <componentId, MkGrouper>>
 */
public static Map<String, Map<String, MkGrouper>> outbound_components(
        TopologyContext topology_context, WorkerData workerData) {
    Map<String, Map<String, MkGrouper>> rr = new HashMap<>();

    // <Stream_id,<component,Grouping>>
    Map<String, Map<String, Grouping>> output_groupings = topology_context.getThisTargets();
    for (Entry<String, Map<String, Grouping>> entry : output_groupings.entrySet()) {
        String stream_id = entry.getKey();
        Map<String, Grouping> component_grouping = entry.getValue();

        Fields out_fields = topology_context.getThisOutputFields(stream_id);

        Map<String, MkGrouper> componentGrouper = new HashMap<>();
        for (Entry<String, Grouping> cg : component_grouping.entrySet()) {
            String component = cg.getKey();
            Grouping tgrouping = cg.getValue();

            List<Integer> outTasks = topology_context.getComponentTasks(component);
            // ATTENTION: If topology set one component parallelism as 0
            // so we don't need send tuple to it
            if (outTasks.size() > 0) {
                MkGrouper grouper = new MkGrouper(
                        topology_context, out_fields, tgrouping, component, stream_id, workerData);
                componentGrouper.put(component, grouper);
            }
            LOG.info("outbound_components, {}-{} for task-{} on {}",
                    component, outTasks, topology_context.getThisTaskId(), stream_id);
        }
        if (componentGrouper.size() > 0) {
            rr.put(stream_id, componentGrouper);
        }
    }
    return rr;
}
 
Example #24
Source File: SequenceSpout.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    
    if (conf.get("spout.max.sending.num") == null) {
        isLimited = false;
    } else {
        isLimited = true;
        SPOUT_MAX_SEND_NUM = JStormUtils.parseLong(conf.get("spout.max.sending.num"));
    }
    
    Boolean btrue = JStormUtils.parseBoolean(conf.get("spout.send.contrl.message"));
    if (btrue != null && btrue) {
        isSendCtrlMsg = true;
    } else {
        isSendCtrlMsg = false;
    }
    
    isFinished = false;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    MetricClient metricClient = new MetricClient(context);
    this.tpCounter = metricClient.registerTopologyCounter("TpCounter");

    MAX_PENDING_COUNTER = getMaxPending(conf);
    
    bufferLen = JStormUtils.parseInt(conf.get("byte.buffer.len"), 0);
    
    random = new Random();
    random.setSeed(System.currentTimeMillis());

    idGenerate = new Random(Utils.secureRandomLong());

    JStormUtils.sleepMs(20 * 1000);
    
    LOG.info("Finish open, buffer Len:" + bufferLen);
}
 
Example #25
Source File: SequenceFileBolt.java    From storm-hdfs with Apache License 2.0 5 votes vote down vote up
@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 
Example #26
Source File: StatefulWindowedBoltExecutor.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private void init(Map stormConf, TopologyContext context, OutputCollector collector,
                  KeyValueState<TaskStream, WindowState> windowState) {
    if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME)) {
        msgIdFieldName = (String) stormConf.get(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME);
    } else {
        throw new IllegalArgumentException(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME + " is not set");
    }
    topologyContext = context;
    outputCollector = collector;
    streamState = windowState;
    pendingTuples = new ArrayList<>();
    recoveryStates = new HashMap<>();
    stateInitialized = false;
    prePrepared = false;
}
 
Example #27
Source File: FeatureExtractionOp.java    From StormCV with Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
protected void prepareOpenCVOp(Map stormConf, TopologyContext context) throws Exception { }
 
Example #28
Source File: FixedBatchSpout.java    From storm-hdfs with Apache License 2.0 4 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context) {
    index = 0;
}
 
Example #29
Source File: BoltCollector.java    From jstorm with Apache License 2.0 4 votes vote down vote up
void unanchoredSend(TopologyContext topologyContext, TaskSendTargets taskTargets,
                    TaskTransfer transfer_fn, String stream, List<Object> values) {
    UnanchoredSend.send(topologyContext, taskTargets, transfer_fn, stream, values);
}
 
Example #30
Source File: SlidingWindowSumBolt.java    From jstorm with Apache License 2.0 4 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
}