Java Code Examples for backtype.storm.task.TopologyContext

The following examples show how to use backtype.storm.task.TopologyContext. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: StormCV   Source File: BatchInputBolt.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
void prepare(Map conf, TopologyContext context) {
	// use TTL and maxSize from config if they were not set explicitly using the constructor (implicit way of doing this...)
	if(TTL == 29) TTL = conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC) == null ? TTL : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC)).intValue();
	if(maxSize == 256) maxSize = conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE) == null ? maxSize : ((Long)conf.get(StormCVConfig.STORMCV_CACHES_MAX_SIZE)).intValue();
	history = new History(this);
	
	// IF NO grouping was set THEN select the first grouping registered for the spout as the grouping used within the Spout (usually a good guess)
	if(groupBy == null){
		Map<GlobalStreamId, Grouping> sources = context.getSources(context.getThisComponentId());
		for(GlobalStreamId id : sources.keySet()){
			Grouping grouping = sources.get(id);
			this.groupBy = new Fields(grouping.get_fields());
			break;
		}
	}
	
	// prepare the selector and operation
	try {
		batcher.prepare(conf);
		operation.prepare(conf, context);
	} catch (Exception e) {
		logger.error("Unable to preapre the Selector or Operation", e);
	}
}
 
Example 2
Source Project: jstorm   Source File: TaskHeartbeatTrigger.java    License: Apache License 2.0 6 votes vote down vote up
public TaskHeartbeatTrigger(Map conf, String name, DisruptorQueue controlQueue, int taskId, String componentId,
                            TopologyContext sysTopologyCtx, ITaskReportErr reportError, TaskStatus executorStatus) {
    this.name = name;
    this.queue = controlQueue;
    this.opCode = TimerConstants.TASK_HEARTBEAT;

    this.taskId = taskId;
    this.componentId = componentId;
    this.sysTopologyCtx = sysTopologyCtx;

    this.frequency = JStormUtils.parseInt(conf.get(Config.TASK_HEARTBEAT_FREQUENCY_SECS), 10);
    this.firstTime = 0;

    this.executeThreadHbTime = TimeUtils.current_time_secs();
    this.taskHbTimeout = JStormUtils.parseInt(conf.get(Config.NIMBUS_TASK_TIMEOUT_SECS), 180);
    this.intervalCheck = new IntervalCheck();
    this.intervalCheck.setInterval(taskHbTimeout);
    this.intervalCheck.start();

    this.reportError = reportError;

    this.uptime = new UptimeComputer();
    this.executorStatus = executorStatus;
}
 
Example 3
Source Project: senti-storm   Source File: SVMBolt.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(Map config, TopologyContext context) {
  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  LOG.info("Loading SVM model...");
  m_dataset = Configuration.getDataSetSemEval2013();
  m_model = SerializationUtils.deserialize(m_dataset.getDatasetPath()
      + File.separator + SVM.SVM_MODEL_FILE_SER);

  if (m_model == null) {
    LOG.error("Could not load SVM model! File: " + m_dataset.getDatasetPath()
        + File.separator + SVM.SVM_MODEL_FILE_SER);
    throw new RuntimeException();
  }
}
 
Example 4
Source Project: eagle   Source File: SystemCheckPersistBolt.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
    this.collector = outputCollector;
    this.client = new EagleServiceClientImpl(config.getConfig());
    if (this.batchSize > 0) {
        this.batchSender = client.batch(this.batchSize);
    }
    this.rackResolver = new DefaultTopologyRackResolver();
    if (config.dataExtractorConfig.resolverCls != null) {
        try {
            rackResolver = config.dataExtractorConfig.resolverCls.newInstance();
            rackResolver.prepare(config);
        } catch (InstantiationException | IllegalAccessException e) {
            LOG.error(e.getMessage(), e);
        }
    }
}
 
Example 5
Source Project: jstorm   Source File: StatefulWindowedBoltExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void init(Map stormConf, TopologyContext context, OutputCollector collector,
                  KeyValueState<TaskStream, WindowState> windowState) {
    if (stormConf.containsKey(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME)) {
        msgIdFieldName = (String) stormConf.get(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME);
    } else {
        throw new IllegalArgumentException(Config.TOPOLOGY_BOLTS_MESSAGE_ID_FIELD_NAME + " is not set");
    }
    topologyContext = context;
    outputCollector = collector;
    streamState = windowState;
    pendingTuples = new ArrayList<>();
    recoveryStates = new HashMap<>();
    stateInitialized = false;
    prePrepared = false;
}
 
Example 6
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    startTime = System.currentTimeMillis();
    maxSendNum = JStormUtils.parseLong(conf.get("max.send.num"), 1000L);
}
 
Example 7
@Override
public void prepare(org.apache.heron.api.topology.TopologyContext context,
                    String component, String streamId,
                    List<Integer> targetTasks) {
  TopologyContext c = new TopologyContext(context);
  GlobalStreamId g = new GlobalStreamId(component, streamId);
  delegate.prepare(c, g, targetTasks);
}
 
Example 8
Source Project: StormCV   Source File: FetchAndOperateFetcher.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	fetcher.prepare(stormConf, context);
	if(sio) singleInOp.prepare(stormConf, context);
	else batchOp.prepare(stormConf, context);
	results = new ArrayList<CVParticle>();
}
 
Example 9
Source Project: incubator-heron   Source File: IRichSpoutDelegate.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void update(org.apache.heron.api.topology.TopologyContext topologyContext) {
  if (delegate instanceof IUpdatable) {
    ((IUpdatable) delegate).update(topologyContext);
  } else {
    LOG.warning(String.format("Update() event received but can not call update() on delegate "
        + "because it does not implement %s: %s", IUpdatable.class.getName(), delegate));
  }
}
 
Example 10
Source Project: jstorm   Source File: TestBackpressure.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.conf = conf;
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = JStormUtils.parseInt(conf.get("send.num.each.time"), 1);
    isStatEnable = JStormUtils.parseBoolean(conf.get("is.stat.enable"), true);
}
 
Example 11
Source Project: jstorm   Source File: KafkaSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
	this.collector = collector;
	if (this.config == null) {
		config = new KafkaSpoutConfig();
	}
	config.configure(conf);
	zkState = new ZkState(conf, config);
	coordinator = new PartitionCoordinator(conf, config, context, zkState);
	lastUpdateMs = System.currentTimeMillis();
}
 
Example 12
Source Project: senti-storm   Source File: FeatureGenerationBolt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map config, TopologyContext context) {
  this.m_dataset = Configuration.getDataSetSemEval2013();

  // Optional set logging
  if (config.get(CONF_LOGGING) != null) {
    m_logging = (Boolean) config.get(CONF_LOGGING);
  } else {
    m_logging = false;
  }

  // TODO serialize CombinedFeatureVectorGenerator
  List<FeaturedTweet> featuredTrainTweets = SerializationUtils
      .deserialize(m_dataset.getTrainDataSerializationFile());
  if (featuredTrainTweets != null) {
    TweetTfIdf tweetTfIdf = TweetTfIdf.createFromTaggedTokens(
        FeaturedTweet.getTaggedTokensFromTweets(featuredTrainTweets),
        TfType.LOG, TfIdfNormalization.COS, true);

    LOG.info("Load CombinedFeatureVectorGenerator...");
    m_fvg = new CombinedFeatureVectorGenerator(true, tweetTfIdf);

  } else {
    LOG.error("TaggedTweets could not be found! File is missing: "
        + m_dataset.getTrainDataSerializationFile());
  }
}
 
Example 13
Source Project: jstorm   Source File: RocksDbKvStoreManagerFactory.java    License: Apache License 2.0 5 votes vote down vote up
public void initMonitor(TopologyContext context) {
    isEnableMetrics = true;
    metricClient = new MetricClient(context);
    rocksDbFlushAndCpLatency = metricClient.registerHistogram("RocksDB flush and checkpoint latency");
    hdfsWriteLatency = metricClient.registerHistogram("HDFS write latency");
    hdfsDeleteLatency = metricClient.registerHistogram("HDFS delete latency");
}
 
Example 14
Source Project: jstorm   Source File: RollingCountBolt.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    lastModifiedTracker = new NthLastModifiedTimeTracker(
            deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds));
}
 
Example 15
Source Project: jstorm   Source File: Common.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * get current task's output <Stream_id, <componentId, MkGrouper>>
 */
public static Map<String, Map<String, MkGrouper>> outbound_components(
        TopologyContext topology_context, WorkerData workerData) {
    Map<String, Map<String, MkGrouper>> rr = new HashMap<>();

    // <Stream_id,<component,Grouping>>
    Map<String, Map<String, Grouping>> output_groupings = topology_context.getThisTargets();
    for (Entry<String, Map<String, Grouping>> entry : output_groupings.entrySet()) {
        String stream_id = entry.getKey();
        Map<String, Grouping> component_grouping = entry.getValue();

        Fields out_fields = topology_context.getThisOutputFields(stream_id);

        Map<String, MkGrouper> componentGrouper = new HashMap<>();
        for (Entry<String, Grouping> cg : component_grouping.entrySet()) {
            String component = cg.getKey();
            Grouping tgrouping = cg.getValue();

            List<Integer> outTasks = topology_context.getComponentTasks(component);
            // ATTENTION: If topology set one component parallelism as 0
            // so we don't need send tuple to it
            if (outTasks.size() > 0) {
                MkGrouper grouper = new MkGrouper(
                        topology_context, out_fields, tgrouping, component, stream_id, workerData);
                componentGrouper.put(component, grouper);
            }
            LOG.info("outbound_components, {}-{} for task-{} on {}",
                    component, outTasks, topology_context.getThisTaskId(), stream_id);
        }
        if (componentGrouper.size() > 0) {
            rr.put(stream_id, componentGrouper);
        }
    }
    return rr;
}
 
Example 16
Source Project: jstorm   Source File: SequenceSpout.java    License: Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    
    if (conf.get("spout.max.sending.num") == null) {
        isLimited = false;
    } else {
        isLimited = true;
        SPOUT_MAX_SEND_NUM = JStormUtils.parseLong(conf.get("spout.max.sending.num"));
    }
    
    Boolean btrue = JStormUtils.parseBoolean(conf.get("spout.send.contrl.message"));
    if (btrue != null && btrue) {
        isSendCtrlMsg = true;
    } else {
        isSendCtrlMsg = false;
    }
    
    isFinished = false;
    
    tpsCounter = new TpsCounter(context.getThisComponentId() + ":" + context.getThisTaskId());
    MetricClient metricClient = new MetricClient(context);
    this.tpCounter = metricClient.registerTopologyCounter("TpCounter");

    MAX_PENDING_COUNTER = getMaxPending(conf);
    
    bufferLen = JStormUtils.parseInt(conf.get("byte.buffer.len"), 0);
    
    random = new Random();
    random.setSeed(System.currentTimeMillis());

    idGenerate = new Random(Utils.secureRandomLong());

    JStormUtils.sleepMs(20 * 1000);
    
    LOG.info("Finish open, buffer Len:" + bufferLen);
}
 
Example 17
Source Project: jstorm   Source File: ForwardingMetricsConsumer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, Object registrationArgument, TopologyContext context, IErrorReporter errorReporter) {
    String[] parts = ((String) registrationArgument).split(":", 2);
    host = parts[0];
    port = Integer.valueOf(parts[1]);
    try {
        socket = new Socket(host, port);
        out = socket.getOutputStream();
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 18
Source Project: storm-hdfs   Source File: SequenceFileBolt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 
Example 19
Source Project: jstorm   Source File: PerformanceTestTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    this.count = 0l;
    this.isAckRequired = Utils.getBoolean(conf.get("is.ack.required"), false);
    this.sendNumEachTime = Utils.getInt(conf.get("send.num.each.time"), 1);
    this.startTimeStamp = System.currentTimeMillis();
}
 
Example 20
Source Project: jstorm   Source File: WindowedBoltExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private Set<GlobalStreamId> getComponentStreams(TopologyContext context) {
    Set<GlobalStreamId> streams = new HashSet<>();
    for (GlobalStreamId streamId : context.getThisSources().keySet()) {
        if (!streamId.get_streamId().equals(CheckpointSpout.CHECKPOINT_STREAM_ID)) {
            streams.add(streamId);
        }
    }
    return streams;
}
 
Example 21
Source Project: jstorm   Source File: JStormUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void reportError(TopologyContext topologyContext, String errorMessge) {
    StormClusterState zkCluster = topologyContext.getZkCluster();
    String topologyId = topologyContext.getTopologyId();
    int taskId = topologyContext.getThisTaskId();

    try {
        zkCluster.report_task_error(topologyId, taskId, errorMessge);
    } catch (Exception e) {
        LOG.warn("Failed to report Error");
    }
}
 
Example 22
Source Project: eagle   Source File: TestHiveJobFetchSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testOpen() throws Exception {
    Map conf = mock(HashMap.class);
    TopologyContext context = mock(TopologyContext.class);
    SpoutOutputCollector collector = mock(SpoutOutputCollector.class);
    when(context.getThisTaskId()).thenReturn(1);
    when(context.getComponentTasks(anyString())).thenReturn(new ArrayList<Integer>() {{
        add(1);
    }});
    HiveJobRunningSourcedStormSpoutProvider provider = new HiveJobRunningSourcedStormSpoutProvider();
    HiveJobFetchSpout spout = (HiveJobFetchSpout)provider.getSpout(config, 1);
    spout.open(conf, context, collector);
    Long yesterday = Long.valueOf(new String(curator.getData().forPath(SHARE_RESOURCES)));
    Assert.assertTrue(System.currentTimeMillis() > yesterday);
}
 
Example 23
Source Project: jstorm   Source File: DRPCSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _clients = new ArrayList<>();
    if (_local_drpc_id == null) {
        _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>());
        _futures = new LinkedList<>();

        int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
        int index = context.getThisTaskIndex();

        int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
        List<String> servers = NetWorkUtils.host2Ip((List<String>) conf.get(Config.DRPC_SERVERS));

        if (servers == null || servers.isEmpty()) {
            throw new RuntimeException("No DRPC servers configured for topology");
        }

        if (numTasks < servers.size()) {
            for (String s : servers) {
                _futures.add(_backround.submit(new Adder(s, port, conf)));
            }
        } else {
            int i = index % servers.size();
            _futures.add(_backround.submit(new Adder(servers.get(i), port, conf)));
        }
    }

}
 
Example 24
Source Project: StormCV   Source File: MjpegStreamingOp.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context)	throws Exception {
	images = MjpegStreamingOp.getImages();
	
	ApplicationAdapter connector = new ApplicationAdapter(new MjpegStreamingOp());
	server = HttpServerFactory.create ("http://localhost:"+port+"/", connector);
	server.start();		
}
 
Example 25
Source Project: cognition   Source File: CsvHdfsBolt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context) {
  try {
    prepareHDFS();
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example 26
Source Project: jstorm   Source File: SequenceFileBolt.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 
Example 27
Source Project: storm-hdfs   Source File: FixedBatchSpout.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context) {
    index = 0;
}
 
Example 28
Source Project: jstorm   Source File: JoinResult.java    License: Apache License 2.0 4 votes vote down vote up
public void prepare(Map map, TopologyContext context, OutputCollector collector) {
    _collector = collector;
}
 
Example 29
Source Project: jstorm   Source File: CatMetricsConsumer.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void prepare(Map stormConf, Object registrationArgument, 
        TopologyContext context, IErrorReporter errorReporter) {
    stormId = context.getStormId();
}
 
Example 30
Source Project: jstorm   Source File: FastWordCountTopology.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    
}