Java Code Examples for org.apache.storm.task.TopologyContext#getThisTaskId()

/**
 * Loads the last saved checkpoint state the from persistent storage.
 */
private KeyValueState<String, CheckPointState> loadCheckpointState(Map<String, Object> conf,
                                                                   TopologyContext ctx) {
  String namespace = ctx.getThisComponentId() + "-" + ctx.getThisTaskId();
  KeyValueState<String, CheckPointState> state
      = (KeyValueState<String, CheckPointState>) StateFactory.getState(namespace, conf, ctx);
  if (state.get(TX_STATE_KEY) == null) {
    CheckPointState txState = new CheckPointState(-1, COMMITTED);
    state.put(TX_STATE_KEY, txState);
    state.commit();
    LOG.fine(() -> String.format("Initialized checkpoint spout state with txState %s", txState));
  } else {
    LOG.fine(() -> String.format("Got checkpoint spout state %s", state.get(TX_STATE_KEY)));
  }
  return state;
}
 

/**
 * 初始化
 */
@Override
public void prepare(Map stormConf, TopologyContext context,
		OutputCollector collector) {
	this.counters = new HashMap<String, Integer>();
	this.collector = collector;
	this.name = context.getThisComponentId();
	this.id = context.getThisTaskId();
}
 

/**
 * 初始化
 */
@Override
public void prepare(Map stormConf, TopologyContext context,
		OutputCollector collector) {
	this.counters = new HashMap<String, Integer>();
	this.collector = collector;
	this.name = context.getThisComponentId();
	this.id = context.getThisTaskId();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collector) {

    super.prepare(stormConf, context, collector);

    Config conf = new Config();
    conf.putAll(stormConf);

    checkConfiguration(conf);

    LOG.info("[Fetcher #{}] : starting at {}", taskID, Instant.now());

    int metricsTimeBucketSecs = ConfUtils.getInt(conf,
            "fetcher.metrics.time.bucket.secs", 10);

    // Register a "MultiCountMetric" to count different events in this bolt
    // Storm will emit the counts every n seconds to a special bolt via a
    // system stream
    // The data can be accessed by registering a "MetricConsumer" in the
    // topology
    this.eventCounter = context.registerMetric("fetcher_counter",
            new MultiCountMetric(), metricsTimeBucketSecs);

    // create gauges
    context.registerMetric("activethreads", () -> {
        return activeThreads.get();
    }, metricsTimeBucketSecs);

    context.registerMetric("in_queues", () -> {
        return fetchQueues.inQueues.get();
    }, metricsTimeBucketSecs);

    context.registerMetric("num_queues", () -> {
        return fetchQueues.queues.size();
    }, metricsTimeBucketSecs);

    this.averagedMetrics = context.registerMetric("fetcher_average_perdoc",
            new MultiReducedMetric(new MeanReducer()),
            metricsTimeBucketSecs);

    this.perSecMetrics = context.registerMetric("fetcher_average_persec",
            new MultiReducedMetric(new PerSecondReducer()),
            metricsTimeBucketSecs);

    protocolFactory = new ProtocolFactory(conf);

    this.fetchQueues = new FetchItemQueues(conf);

    this.taskID = context.getThisTaskId();

    int threadCount = ConfUtils.getInt(conf, "fetcher.threads.number", 10);
    for (int i = 0; i < threadCount; i++) { // spawn threads
        new FetcherThread(conf, i).start();
    }

    // keep track of the URLs in fetching
    beingFetched = new String[threadCount];
    Arrays.fill(beingFetched, "");

    sitemapsAutoDiscovery = ConfUtils.getBoolean(stormConf,
            SITEMAP_DISCOVERY_PARAM_KEY, false);

    maxNumberURLsInQueues = ConfUtils.getInt(conf,
            "fetcher.max.urls.in.queues", -1);

    /**
     * If set to a valid path e.g. /tmp/fetcher-dump-{port} on a worker
     * node, the content of the queues will be dumped to the logs for
     * debugging. The port number needs to match the one used by the
     * FetcherBolt instance.
     **/
    String debugfiletriggerpattern = ConfUtils.getString(conf,
            "fetcherbolt.queue.debug.filepath");

    if (StringUtils.isNotBlank(debugfiletriggerpattern)) {
        debugfiletrigger = new File(
                debugfiletriggerpattern.replaceAll("\\{port\\}",
                        Integer.toString(context.getThisWorkerPort())));
    }

}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collector) {
    super.prepare(stormConf, context, collector);
    this.conf = new Config();
    this.conf.putAll(stormConf);

    checkConfiguration();

    this.taskID = context.getThisTaskId();

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss",
            Locale.ENGLISH);
    long start = System.currentTimeMillis();
    LOG.info("[Fetcher #{}] : starting at {}", taskID, sdf.format(start));

    // Register a "MultiCountMetric" to count different events in this bolt
    // Storm will emit the counts every n seconds to a special bolt via a
    // system stream
    // The data can be accessed by registering a "MetricConsumer" in the
    // topology

    int metricsTimeBucketSecs = ConfUtils.getInt(conf,
            "fetcher.metrics.time.bucket.secs", 10);

    this.eventCounter = context.registerMetric("fetcher_counter",
            new MultiCountMetric(), metricsTimeBucketSecs);

    this.averagedMetrics = context.registerMetric("fetcher_average",
            new MultiReducedMetric(new MeanReducer()),
            metricsTimeBucketSecs);

    this.perSecMetrics = context.registerMetric("fetcher_average_persec",
            new MultiReducedMetric(new PerSecondReducer()),
            metricsTimeBucketSecs);

    // create gauges
    context.registerMetric("activethreads", new IMetric() {
        @Override
        public Object getValueAndReset() {
            return activeThreads.get();
        }
    }, metricsTimeBucketSecs);

    context.registerMetric("throttler_size", new IMetric() {
        @Override
        public Object getValueAndReset() {
            return throttler.size();
        }
    }, metricsTimeBucketSecs);

    protocolFactory = new ProtocolFactory(conf);

    sitemapsAutoDiscovery = ConfUtils.getBoolean(stormConf,
            SITEMAP_DISCOVERY_PARAM_KEY, false);

    queueMode = ConfUtils.getString(conf, "fetcher.queue.mode",
            QUEUE_MODE_HOST);
    // check that the mode is known
    if (!queueMode.equals(QUEUE_MODE_IP)
            && !queueMode.equals(QUEUE_MODE_DOMAIN)
            && !queueMode.equals(QUEUE_MODE_HOST)) {
        LOG.error("Unknown partition mode : {} - forcing to byHost",
                queueMode);
        queueMode = QUEUE_MODE_HOST;
    }
    LOG.info("Using queue mode : {}", queueMode);

    this.crawlDelay = (long) (ConfUtils.getFloat(conf,
            "fetcher.server.delay", 1.0f) * 1000);

    this.maxCrawlDelay = (long) ConfUtils.getInt(conf,
            "fetcher.max.crawl.delay", 30) * 1000;

    this.maxCrawlDelayForce = ConfUtils.getBoolean(conf,
            "fetcher.max.crawl.delay.force", false);
    this.crawlDelayForce = ConfUtils.getBoolean(conf,
            "fetcher.server.delay.force", false);

    this.maxThrottleSleepMSec = ConfUtils.getLong(conf,
            "fetcher.max.throttle.sleep", -1);

    this.protocolMDprefix = ConfUtils.getString(conf,
            ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, protocolMDprefix);
}