backtype.storm.task.OutputCollector Java Examples

public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
  this.collector = collector;
  this.conf = stormConf;
  if (conf.containsKey("bolt.parser.num.of.key.chars.to.use.for.shuffle.grouping")) {
    this.numberOfCharsToUseForShuffleGrouping = Integer.valueOf(conf.get(
        "bolt.parser.num.of.key.chars.to.use.for.shuffle.grouping").toString());
  }
  
  Grouping._Fields a;


  // hBaseStreamPartitioner = new HBaseStreamPartitioner(
  // conf.get("bolt.hbase.table.name").toString(),
  // 0,
  // Integer.parseInt(conf.get("bolt.hbase.partitioner.region.info.refresh.interval.mins").toString()))
  // ;
  // hBaseStreamPartitioner.prepare();

}
 

@Override
public void receive(long timeStamp, Event[] inEvents, Event[] removeEvents) {
    Object[] attrValues = inEvents[0].getData();
    OutputCollector collector = (OutputCollector) attrValues[0];
    SortedMap<String, Object> outputEvent = new TreeMap<String, Object>();
    int i = 1;  // output is from second element
    String user = null;
    for(String attrKey : outputSelector.keySet()){
        Object v = attrValues[i++];
        outputEvent.put(attrKey, v);
        if(attrKey.equals("user"))
            user = (String)v;
    }

    outputEvent.putAll(outputModifier);
    LOG.debug("outputEvent: " + outputEvent);
    collector.emit(Arrays.asList(user, outputEvent));
}
 

@Override
public final void prepare(final Map map, final TopologyContext topologyContext,
                          final OutputCollector outputCollector) {
	this._outputCollector = outputCollector;

	final Properties properties = new Properties();
	try {
		properties.load(StateLocatorBolt.class.getClassLoader()
				                .getResourceAsStream(Constants.CONFIG_PROPERTIES_FILE));
	} catch (final IOException ioException) {
		//Should not occur. If it does, we cant continue. So exiting the program!
		LOGGER.error(ioException.getMessage(), ioException);
		System.exit(1);
	}
	//Bolt reads the Bing Maps API Value and stores the same to BING_MAPS_API_KEY_VALUE of Constants.java so that the Bolt can use it.
	//For the lack of time I am using this Constant or else using a good Design Pattern, this can be fine-tuned.
	Constants.BING_MAPS_API_KEY_VALUE = properties.getProperty(Constants.BING_MAPS_API_KEY);
}
 

@Override
void doPrepare(Map conf, TopologyContext topologyContext,
		OutputCollector collector) throws IOException {
	LOG.info("[OpenSOC] Preparing Enrichment Bolt...");

	_collector = collector;

	try {
		_reporter = new MetricReporter();
		_reporter.initialize(metricConfiguration,
				GenericEnrichmentBolt.class);
		this.registerCounters();
	} catch (Exception e) {
		LOG.info("[OpenSOC] Unable to initialize metrics reporting");
	}

	LOG.info("[OpenSOC] Enrichment bolt initialized...");
}
 

@Override
public final void prepare(final Map map, final TopologyContext topologyContext,
                          final OutputCollector outputCollector) {
	this._outputCollector = outputCollector;

	final Properties properties = new Properties();
	try {
		properties.load(StateLocatorBolt.class.getClassLoader()
				                .getResourceAsStream(Constants.CONFIG_PROPERTIES_FILE));
	} catch (final IOException ioException) {
		//Should not occur. If it does, we cant continue. So exiting the program!
		LOGGER.error(ioException.getMessage(), ioException);
		System.exit(1);
	}
	//Bolt reads the Bing Maps API Value and stores the same to BING_MAPS_API_KEY_VALUE of Constants.java so that the Bolt can use it.
	//For the lack of time I am using this Constant or else using a good Design Pattern, this can be fine-tuned.
	//For Bing Maps Key: https://www.bingmapsportal.com
	Constants.BING_MAPS_API_KEY_VALUE = properties.getProperty(Constants.BING_MAPS_API_KEY);
}
 

public ExecuteResult execute(Tuple input, OutputCollector outputCollector) {

    if (tuplesReceived != null) {
      tuplesReceived.inc();
    }

    String docId = input.getString(0);
    Object docObj = input.getValue(1);
    if (docId == null || docObj == null) {

      log.warn("Ignored tuple: "+input);

      return ExecuteResult.IGNORED; // nothing to index
    }

    try {
      return processInputDoc(docId, docObj);
    } catch (Exception exc) {
      log.error("Failed to process "+docId+" due to: "+exc);
      if (exc instanceof RuntimeException) {
        throw (RuntimeException)exc;
      } else {
        throw new RuntimeException(exc);
      }
    }
  }
 

public final void prepare(Map conf, TopologyContext topologyContext,
		OutputCollector collector) {
	_collector = collector;

	if (this._IndexIP == null)
		throw new IllegalStateException("_IndexIP must be specified");
	if (this._IndexPort == 0)
		throw new IllegalStateException("_IndexPort must be specified");
	if (this._ClusterName == null)
		throw new IllegalStateException("_ClusterName must be specified");
	if (this._IndexName == null)
		throw new IllegalStateException("_IndexName must be specified");
	if (this._DocumentName == null)
		throw new IllegalStateException("_DocumentName must be specified");
	if (this._adapter == null)
		throw new IllegalStateException("IndexAdapter must be specified");

	try {
		doPrepare(conf, topologyContext, collector);
	} catch (IOException e) {
		LOG.error("Counld not initialize...");
		e.printStackTrace();
	}
}
 

@Override
void doPrepare(Map conf, TopologyContext topologyContext,
		OutputCollector collector) throws IOException {

	LOG.info("[OpenSOC] Preparing TelemetryParser Bolt...");

	if (metricConfiguration != null) {
		_reporter = new MetricReporter();
		_reporter
				.initialize(metricConfiguration, TelemetryParserBolt.class);
		LOG.info("[OpenSOC] Metric reporter is initialized");
	} else {
		LOG.info("[OpenSOC] Metric reporter is not initialized");
	}
	this.registerCounters();
	
	if(_parser != null)
	_parser.init();
	
	
}
 

/**
 * {@inheritDoc}
 */
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector)
{
    LOG.debug("Start to prepare storm output bolt.");
    outputCollector = collector;
    if (null != output.getConfig().get(StreamingConfig.STREAMING_STORM_COMMON_ISACK))
    {
        needAck = Boolean.valueOf(output.getConfig().get(StreamingConfig.STREAMING_STORM_COMMON_ISACK).toString());
    }
    try
    {
        output.initialize();
    }
    catch (StreamingException e)
    {
        LOG.error("Failed to initialize output stream.");
        throw new RuntimeException("Failed to initialize output stream", e);
    }
}
 

public void prepare(Map config, TopologyContext context, OutputCollector collector) {
    TimeCacheMap.ExpiredCallback<Object, TrackingInfo> callback = null;
    if (_delegate instanceof TimeoutCallback) {
        callback = new TimeoutItems();
    }
    _tracked = new TimeCacheMap<>(context.maxTopologyMessageTimeout(), callback);
    _collector = collector;
    _delegate.prepare(config, context, new OutputCollector(new CoordinatedOutputCollector(collector)));
    for (String component : Utils.get(context.getThisTargets(), Constants.COORDINATED_STREAM_ID, new HashMap<String, Grouping>()).keySet()) {
        for (Integer task : context.getComponentTasks(component)) {
            _countOutTasks.add(task);
        }
    }
    if (!_sourceArgs.isEmpty()) {
        _numSourceReports = 0;
        for (Entry<String, SourceArgs> entry : _sourceArgs.entrySet()) {
            if (entry.getValue().singleCount) {
                _numSourceReports += 1;
            } else {
                _numSourceReports += context.getComponentTasks(entry.getKey()).size();
            }
        }
    }
}
 

@Override
void doPrepare(Map conf, TopologyContext topologyContext,
		OutputCollector collector) throws IOException {

	cache = CacheBuilder.newBuilder().maximumSize(_MAX_CACHE_SIZE_OBJECTS_NUM)
			.expireAfterWrite(_MAX_TIME_RETAIN_MINUTES, TimeUnit.MINUTES).build();

	LOG.info("[OpenSOC] Preparing TelemetryAlert Bolt...");

	try {
		_reporter = new MetricReporter();
		_reporter.initialize(metricProperties, TelemetryAlertsBolt.class);
		LOG.info("[OpenSOC] Initialized metrics");
	} catch (Exception e) {
		LOG.info("[OpenSOC] Could not initialize metrics");
	}
}
 

@Override
public void prepare(Map map, TopologyContext topologyContext, OutputCollector outputCollector) {
    this.collector = outputCollector;
    this.hdfsConf = new Configuration();
    this.hdfsConf.setBoolean("fs.hdfs.impl.disable.cache", true);
    for (Map.Entry<String, String> entry : config.jobHistoryConfig.hdfs.entrySet()) {
        this.hdfsConf.set(entry.getKey(), entry.getValue());
        LOG.info("conf key {}, conf value {}", entry.getKey(), entry.getValue());
    }
    this.zkState = new JobHistoryZKStateManager(config);
}
 

@Override
@SuppressWarnings("rawtypes")
public void prepare(
    Map conf,
    TopologyContext context,
    OutputCollector acollector) {
  collector = acollector;
  nItems = 0;
  startTime = System.currentTimeMillis();

  // Add task hook dynamically
  context.addTaskHook(new TestTaskHook("This TestTaskHook is constructed by CountBolt"));

}
 

private KafkaBolt generateDefaultSerializerBolt() {
    KafkaBolt bolt = new KafkaBolt();
    Properties props = new Properties();
    props.put("metadata.broker.list", broker.getBrokerConnectionString());
    props.put("request.required.acks", "1");
    config.put(KafkaBolt.KAFKA_BROKER_PROPERTIES, props);
    bolt.prepare(config, null, new OutputCollector(collector));
    return bolt;
}
 

@Override
public void prepare(Map map, TopologyContext topologyContext,
    OutputCollector outputCollector) {
  try {
    this.collector = outputCollector;
    synchronized (EsAbstractBolt.class) {
      if (client == null) {
        client = new StormEsClient(esConfig);
      }
    }
  } catch (Exception e) {
    throw new RuntimeException("unable to initialize EsBolt ", e);
  }
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    metricClient = new MetricClient(context);
    executeCounter = metricClient.registerCounter(SequenceTestMetricsDef.METRIC_TOTAL_EXECUTE);
    tradeSumCounter = metricClient.registerCounter(SequenceTestMetricsDef.METRIC_TOTAL_TRADE_SUM);
    customerSumCounter = metricClient.registerCounter(SequenceTestMetricsDef.METRIC_TOTAL_CUSTOMER_SUM);
}
 

public void prepare(Map stormConf, TopologyContext context,
    OutputCollector collector) {
  collector = this.collector;
  try {
    fos = new FileOutputStream(
        "src/test/resources/test-query.txt");
  } catch (FileNotFoundException e) {
    e.printStackTrace();
  }
}
 

@Override
public void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException {
    LOG.info("Preparing Sequence File Bolt...");
    if (this.format == null) throw new IllegalStateException("SequenceFormat must be specified.");

    this.fs = FileSystem.get(URI.create(this.fsUrl), hdfsConfig);
    this.codecFactory = new CompressionCodecFactory(hdfsConfig);
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _fieldLocations = new HashMap<String, GlobalStreamId>();
    _collector = collector;
    int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
    _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
    _numSources = context.getThisSources().size();
    Set<String> idFields = null;
    for (GlobalStreamId source : context.getThisSources().keySet()) {
        Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
        Set<String> setFields = new HashSet<String>(fields.toList());
        if (idFields == null)
            idFields = setFields;
        else
            idFields.retainAll(setFields);

        for (String outfield : _outFields) {
            for (String sourcefield : fields) {
                if (outfield.equals(sourcefield)) {
                    _fieldLocations.put(outfield, source);
                }
            }
        }
    }
    _idFields = new Fields(new ArrayList<String>(idFields));

    if (_fieldLocations.size() != _outFields.size()) {
        throw new RuntimeException("Cannot find all outfields among sources");
    }
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    registry = new MetricRegistry();
    String host = EagleConfigHelper.getServiceHost(config);
    int port = EagleConfigHelper.getServicePort(config);
    String username = EagleConfigHelper.getServiceUser(config);
    String password = EagleConfigHelper.getServicePassword(config);
    listener = new EagleServiceReporterMetricListener(host, port, username, password);
    dimensions = new HashMap<>();
    dimensions.put(EagleConfigConstants.SITE, config.getString("siteId"));
    dimensions.put(EagleConfigConstants.APPLICATION, config.getString("appId"));
    gcPausedTimeMetricName = MetricKeyCodeDecoder.codeMetricKey(GCConstants.GC_PAUSE_TIME_METRIC_NAME, dimensions);

    this.collector = collector;
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    _messageTimeoutMs = context.maxTopologyMessageTimeout() * 1000L;
    _lastRotate = System.currentTimeMillis();
    _batches = new RotatingMap<>(2);
    _context = context;
    _collector = collector;
    _coordCollector = new CoordinatedOutputCollector(new OutputCollector(collector));
    _coordOutputCollector = new BatchOutputCollectorImpl(new OutputCollector(_coordCollector));

    _coordConditions = (Map) context.getExecutorData("__coordConditions");
    if (_coordConditions == null) {
        _coordConditions = new HashMap<>();
        for (String batchGroup : _coordSpecs.keySet()) {
            CoordSpec spec = _coordSpecs.get(batchGroup);
            CoordCondition cond = new CoordCondition();
            cond.commitStream = spec.commitStream;
            cond.expectedTaskReports = 0;
            for (String comp : spec.coords.keySet()) {
                CoordType ct = spec.coords.get(comp);
                if (ct.equals(CoordType.single())) {
                    cond.expectedTaskReports += 1;
                } else {
                    cond.expectedTaskReports += context.getComponentTasks(comp).size();
                }
            }
            cond.targetTasks = new HashSet<>();
            for (String component : Utils.get(context.getThisTargets(),
                    COORD_STREAM(batchGroup),
                    new HashMap<String, Grouping>()).keySet()) {
                cond.targetTasks.addAll(context.getComponentTasks(component));
            }
            _coordConditions.put(batchGroup, cond);
        }
        context.setExecutorData("_coordConditions", _coordConditions);
    }
    _bolt.prepare(conf, context, _coordOutputCollector);
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    lastModifiedTracker = new NthLastModifiedTimeTracker(
            deriveNumWindowChunksFrom(this.windowLengthInSeconds, this.emitFrequencyInSeconds));
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.batchXorTracker = new AckPendingBatchTracker<>();
    this.ackOutputCollector = new AckOutputCollector(collector.getDelegate(), batchXorTracker);
    this.bolt.prepare(stormConf, context, new OutputCollector(ackOutputCollector));

    this.componentId = context.getThisComponentId();
    this.taskId = context.getThisTaskId();
    this.taskStats = new TaskBaseMetric(context.getTopologyId(), componentId, taskId);
    this.batchTimeout = ConfigExtension.getTransactionBatchSnapshotTimeout(stormConf) * 1000;
    this.random = new Random(Utils.secureRandomLong());

    LOG.info("batchTimeout: {}", batchTimeout);
}
 

@Override
public void internalPrepare(OutputCollector collector, IMetadataChangeNotifyService coordinatorService, Config config, TopologyContext context) {
    coordinatorService.registerListener(this);
    coordinatorService.init(config, MetadataType.ALERT_PUBLISH_BOLT);
    this.alertPublisher.init(config, stormConf);
    streamContext = new StreamContextImpl(config, context.registerMetric("eagle.publisher", new MultiCountMetric(), 60), context);
    this.context = context;
    this.alertTemplateEngine = AlertTemplateProvider.createAlertTemplateEngine();
    this.alertTemplateEngine.init(config);
    this.alertFilter = new PipeStreamFilter(new AlertContextEnrichFilter(this), new AlertTemplateFilter(alertTemplateEngine));
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    Preconditions.checkNotNull(this.changeNotifyService, "IMetadataChangeNotifyService is not set yet");
    this.stormConf = stormConf;
    this.collector = collector;
    this.serializer = Serializers.newPartitionedEventSerializer(this);
    internalPrepare(collector, this.changeNotifyService, this.config, context);
    try {
        this.changeNotifyService.activateFetchMetaData();
    } catch (Exception e) {
        LOG.warn(e.getMessage(), e);
    }
}
 

@Override
public void internalPrepare(OutputCollector collector, IMetadataChangeNotifyService changeNotifyService, Config config, TopologyContext context) {
    streamContext = new StreamContextImpl(config, context.registerMetric("eagle.router", new MultiCountMetric(), 60), context);
    routeCollector = new StreamRouterBoltOutputCollector(getBoltId(), new StormOutputCollector(collector, serializer), this.getOutputStreamIds(), streamContext);
    router.prepare(streamContext, routeCollector);
    changeNotifyService.registerListener(this);
    changeNotifyService.init(config, MetadataType.STREAM_ROUTER_BOLT);
}
 

@SuppressWarnings("unchecked")
@Override
public void execute(Tuple input, OutputCollector collector) {
	// TODO Auto-generated method stub
	Set<String> queues = (Set<String>) input.getValueByField("queue");
	for(String queue : queues){
		new Thread(ConsumeDefinition.findByQueue(queue)).start();
	}
	collector.ack(input);
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    Map configMap = (Map) stormConf.get(KAFKA_BROKER_PROPERTIES);
    Properties properties = new Properties();
    properties.putAll(configMap);
    ProducerConfig config = new ProducerConfig(properties);
    producer = new Producer<K, V>(config);
    this.topic = (String) stormConf.get(TOPIC);
    this.collector = collector;
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.collector = collector;
    lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
            this.emitFrequencyInSeconds));

    jedis = new Jedis(elasticCacheRedisEndpoint);
}
 

public static AlertBolt createAlertBolt(OutputCollector collector) {
    Config config = ConfigFactory.load();
    PolicyGroupEvaluator policyGroupEvaluator = new PolicyGroupEvaluatorImpl("testPolicyGroupEvaluatorImpl");
    TestStreamRouterBolt.MockChangeService mockChangeService = new TestStreamRouterBolt.MockChangeService();
    AlertBolt bolt = new AlertBolt("alertBolt1", config, mockChangeService);
    Map stormConf = new HashMap<>();
    TopologyContext topologyContext = mock(TopologyContext.class);
    when(topologyContext.registerMetric(any(String.class), any(MultiCountMetric.class), any(int.class))).thenReturn(new MultiCountMetric());
    bolt.prepare(stormConf, topologyContext, collector);
    return bolt;
}