org.apache.storm.task.OutputCollector Java Examples

/**
 * Retrieves the ProfileMeasurement(s) (if any) that have been emitted.
 *
 * @param collector The Storm output collector.
 * @param expected The number of measurements expected.
 * @return A list of ProfileMeasurement(s).
 */
private List<ProfileMeasurement> getProfileMeasurements(OutputCollector collector, int expected) {

  // the 'streamId' is defined by the DestinationHandler being used by the bolt
  final String streamId = emitter.getStreamId();

  // capture the emitted tuple(s)
  ArgumentCaptor<Values> argCaptor = ArgumentCaptor.forClass(Values.class);
  verify(collector, times(expected))
          .emit(eq(streamId), argCaptor.capture());

  // return the profile measurements that were emitted
  return argCaptor.getAllValues()
          .stream()
          .map(val -> (ProfileMeasurement) val.get(0))
          .collect(Collectors.toList());
}
 

public void testLagging() {
  final OutputCollector collector = mock(OutputCollector.class);

  final MockMetricFilteringBolt bolt =
      createBolt(new ArrayList<AlarmDefinition>(0), new ArrayList<Alarm>(0), collector, true);

  final long prepareTime = bolt.getCurrentTime();
  final MetricDefinition metricDefinition =
      alarmDef1.getAlarmExpression().getSubExpressions().get(0).getMetricDefinition();
  final long oldestTimestamp = prepareTime - MetricFilteringBolt.LAG_MESSAGE_PERIOD_DEFAULT;
  final Tuple lateMetricTuple =
      createMetricTuple(metricDefinition, oldestTimestamp, new Metric(metricDefinition,
          oldestTimestamp, 42.0, null));
  bolt.execute(lateMetricTuple);
  verify(collector, times(1)).ack(lateMetricTuple);
  bolt.setCurrentTime(prepareTime + MetricFilteringBolt.LAG_MESSAGE_PERIOD_DEFAULT);
  final Tuple lateMetricTuple2 =
      createMetricTuple(metricDefinition, prepareTime, new Metric(metricDefinition, prepareTime,
          42.0, null));
  bolt.execute(lateMetricTuple2);
  verify(collector, times(1)).ack(lateMetricTuple2);
  verify(collector, times(1)).emit(MetricAggregationBolt.METRIC_AGGREGATION_CONTROL_STREAM,
      new Values(MetricAggregationBolt.METRICS_BEHIND));
  bolt.setCurrentTime(prepareTime + 2 * MetricFilteringBolt.LAG_MESSAGE_PERIOD_DEFAULT);
  long caughtUpTimestamp = bolt.getCurrentTime() - MetricFilteringBolt.MIN_LAG_VALUE_DEFAULT;
  final Tuple metricTuple =
      createMetricTuple(metricDefinition, caughtUpTimestamp, new Metric(metricDefinition,
          caughtUpTimestamp, 42.0, null));
  bolt.execute(metricTuple);
  // Metrics are caught up so there should not be another METRICS_BEHIND message
  verify(collector, times(1)).ack(metricTuple);
  verify(collector, times(1)).emit(MetricAggregationBolt.METRIC_AGGREGATION_CONTROL_STREAM,
      new Values(MetricAggregationBolt.METRICS_BEHIND));
}
 

protected void handleError(String sensorType, byte[] originalMessage, Tuple tuple, Throwable ex, OutputCollector collector) {
  MetronError error = new MetronError()
          .withErrorType(Constants.ErrorType.PARSER_ERROR)
          .withThrowable(ex)
          .withSensorType(Collections.singleton(sensorType))
          .addRawMessage(originalMessage);
  handleError(collector, error);
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    // Add the Storm Config and the context as is, in case any PubSubs need it.
    config.set(BulletStormConfig.STORM_CONFIG, conf);
    config.set(BulletStormConfig.STORM_CONTEXT, context);

    this.collector = collector;
    try {
        this.publisher = createPublisher();
    } catch (PubSubException e) {
        throw new RuntimeException("Cannot create PubSub instance or a Publisher for it.", e);
    }
}
 

/**
   * 在Bolt启动前执行，提供Bolt启动环境配置的入口
   * 一般对于不可序列化的对象进行实例化。
   * 注:如果是可以序列化的对象，那么最好是使用构造函数。
   */
@SuppressWarnings("rawtypes")
@Override
public void prepare(Map map, TopologyContext arg1, OutputCollector collector) {
	System.out.println("prepare:"+map.get("test"));
	this.collector=collector;
}
 

@Test
public void testSitemapSubdocuments() throws IOException {
    Map config = new HashMap();
    // generate a dummy config file
    config.put("urlfilters.config.file", "basicurlnormalizer.json");
    bolt.prepare(config, TestUtil.getMockedTopologyContext(),
            new OutputCollector(output));
    Metadata metadata = new Metadata();
    parse("http://www.digitalpebble.com/duplicates.html",
            "duplicateLinks.html", metadata);
    Assert.assertEquals(1, output.getEmitted(Constants.StatusStreamName)
            .size());
}
 

@Override
@SuppressWarnings("rawtypes")
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
  logger = LoggerFactory.getLogger(Logging.categoryFor(getClass(), context));
  logger.info("Preparing");
  this.collector = collector;

  if (this.alarmDAO == null) {
    Injector.registerIfNotBound(AlarmDAO.class, new PersistenceModule(this.dbConfig));
    this.alarmDAO = Injector.getInstance(AlarmDAO.class);
  }
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map conf, TopologyContext context,
        OutputCollector collector) {
    super.prepare(conf, context, collector);
    _collector = collector;
    fields = new HashMap<>();
}
 

/**
 * Handles a {@link MetronError} that occurs.
 *
 * @param collector The Storm output collector being reported to
 * @param error The error that occurred
 */
public static void handleError(OutputCollector collector, MetronError error)
{
  collector.emit(Constants.ERROR_STREAM, new Values(error.getJSONObject()));
  Optional<Throwable> throwable = error.getThrowable();
  if (throwable.isPresent()) {
    collector.reportError(throwable.get());
  }

}
 

@Override
@SuppressWarnings({ "rawtypes" })
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collect) {
    super.prepare(stormConf, context, collect);
    parseFilters = ParseFilters.fromConf(stormConf);
}
 

@Override
public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
    try {
        this.context = context;
        this.collector = collector;
        inner = new SinkerBaseMap(conf);
        init();
        logger.info("[write bolt] init completed.");
    } catch (Exception e) {
        logger.error("[write bolt] init error.", e);
        collector.reportError(e);
    }
}
 

@Override
@SuppressWarnings({ "rawtypes", "unchecked" })
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collect) {
    super.prepare(stormConf, context, collect);
    parseFilters = ParseFilters.fromConf(stormConf);
}
 

/**
 * Used only for unit testing.
 */
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector, Clock clock) {
  prepare(stormConf, context, collector);
  BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(maxBatchTimeout, clock);
  bulkWriterComponent.addFlushPolicy(ackTuplesPolicy);
  setWriterComponent(bulkWriterComponent);
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
  this.collector = collector;
  super.prepare(stormConf, context, collector);
  if (messageGetField != null) {
    messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get(messageGetField);
  } else {
    messageGetStrategy = MessageGetters.valueOf(messageGetStrategyType).get();
  }
  if(bulkMessageWriter instanceof WriterToBulkWriter) {
    configurationTransformation = WriterToBulkWriter.TRANSFORMATION;
  }
  else {
    configurationTransformation = x -> x;
  }
  ackTuplesPolicy = new AckTuplesPolicy(collector, messageGetStrategy);
  try {
    WriterConfiguration writerconf = configurationTransformation
        .apply(getConfigurationStrategy().createWriterConfig(bulkMessageWriter, getConfigurations()));
    if (maxBatchTimeout == 0) {
      //This means getComponentConfiguration was never called to initialize maxBatchTimeout,
      //probably because we are in a unit test scenario.  So calculate it here.
      BatchTimeoutHelper timeoutHelper = new BatchTimeoutHelper(writerconf::getAllConfiguredTimeouts, batchTimeoutDivisor);
      maxBatchTimeout = timeoutHelper.getMaxBatchTimeout();
    }
    BulkWriterComponent<JSONObject> bulkWriterComponent = new BulkWriterComponent<>(maxBatchTimeout);
    bulkWriterComponent.addFlushPolicy(ackTuplesPolicy);
    setWriterComponent(bulkWriterComponent);
    bulkMessageWriter.init(stormConf, writerconf);
    if (bulkMessageWriter instanceof HdfsWriter) {
      ((HdfsWriter) bulkMessageWriter).initFileNameFormat(context);
    }
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 

@Override
public void prepare(Map<String, Object> conf, TopologyContext context, OutputCollector collector) {
    _fieldLocations = new HashMap<String, GlobalStreamId>();
    _collector = collector;
    int timeout = ((Number) conf.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS)).intValue();
    _pending = new TimeCacheMap<List<Object>, Map<GlobalStreamId, Tuple>>(timeout, new ExpireCallback());
    _numSources = context.getThisSources().size();
    Set<String> idFields = null;
    for (GlobalStreamId source : context.getThisSources().keySet()) {
        Fields fields = context.getComponentOutputFields(source.get_componentId(), source.get_streamId());
        Set<String> setFields = new HashSet<String>(fields.toList());
        if (idFields == null) {
            idFields = setFields;
        } else {
            idFields.retainAll(setFields);
        }

        for (String outfield : _outFields) {
            for (String sourcefield : fields) {
                if (outfield.equals(sourcefield)) {
                    _fieldLocations.put(outfield, source);
                }
            }
        }
    }
    _idFields = new Fields(new ArrayList<String>(idFields));

    if (_fieldLocations.size() != _outFields.size()) {
        throw new RuntimeException("Cannot find all outfields among sources");
    }
}
 

private void deleteSubAlarms(MetricFilteringBolt bolt, OutputCollector collector,
    final Alarm alarm) {
  for (final MetricDefinitionAndTenantId mtid : alarm.getAlarmedMetrics()) {
    final Tuple tuple = createMetricDefinitionDeletionTuple(mtid, alarm.getAlarmDefinitionId());
    bolt.execute(tuple);
    verify(collector, times(1)).ack(tuple);
  }
}
 

@Override
protected void handleError(OutputCollector collector, MetronError error) {
  for(Object rawMessage: error.getRawMessages()) {
    errors.add((byte[]) rawMessage);
  }
  if (error.getThrowable().isPresent()) {
    Throwable throwable = error.getThrowable().get();
    LOG.error("Error parsing message: " + throwable.getMessage(), throwable);
  }

}
 

@Test
public void testFeedParsingDetextBytes() throws IOException {

    Map parserConfig = new HashMap();
    parserConfig.put("feed.sniffContent", true);
    parserConfig.put("parsefilters.config.file", "test.parsefilters.json");
    bolt.prepare(parserConfig, TestUtil.getMockedTopologyContext(),
            new OutputCollector(output));

    Metadata metadata = new Metadata();
    parse("http://www.guardian.com/feed.xml", "guardian.rss", metadata);

    checkOutput();
}
 

/**
 * @param batchSize The batch size.
 * @param collector The output collector.
 */
public BatchHelper(int batchSize, OutputCollector collector) {
  if (batchSize > 0) {
    this.batchSize = batchSize;
  }
  this.collector = collector;
  this.tupleBatch = new LinkedList<>();
}
 

@Test
public void handleErrorShouldEmitAndReportError() {
  Throwable e = new Exception("error");
  MetronError error = new MetronError().withMessage("error message").withThrowable(e);
  OutputCollector collector = mock(OutputCollector.class);

  StormErrorUtils.handleError(collector, error);
  verify(collector, times(1)).emit(eq(Constants.ERROR_STREAM), argThat(new MetronErrorJSONMatcher(error.getJSONObject())));
  verify(collector, times(1)).reportError(any());
}
 

/**
 * Creates and initializes a Publisher that writes to the DRPC servers. Intended to be used inside a Storm
 * bolt in a Storm topology.
 *
 * @param config Needs the Storm configuration {@link Map} in {@link com.yahoo.bullet.storm.BulletStormConfig#STORM_CONFIG}.
 */
public DRPCResultPublisher(BulletConfig config) {
    // Get the Storm Config that has all the relevant cluster settings and properties
    Map stormConfig = config.getRequiredConfigAs(DRPCConfig.STORM_CONFIG, Map.class);

    collector = new DRPCOutputCollector();
    // Wrap the collector in a OutputCollector (it just delegates to the underlying DRPCOutputCollector)
    OutputCollector boltOutputCollector = new OutputCollector(collector);

    bolt = new ReturnResults();
    // No need for a TopologyContext
    bolt.prepare(stormConfig, null, boltOutputCollector);
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) {
    eplManager = new EPLManager(stormConf, collector, new CountDownLatch(1));
    this.collector = collector;

    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) {
    eplManager = new EPLManager(stormConf, collector, new CountDownLatch(1));
    this.collector = collector;

    try {
        Thread.sleep(3000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}
 

@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
    this.conf = stormConf;
    this.collector = collector;
    this.context = context;

    //only init one time
    zkServers = (String) stormConf.get(Constants.ZOOKEEPER_SERVERS);
    topologyID = (String) stormConf.get(Constants.TOPOLOGY_ID);
    topologyRoot = Constants.TOPOLOGY_ROOT + "/" + topologyID;

    reloadConfig();

    logger.info("KafkaProducerBolt reload config success !");
}
 

@BeforeMethod
protected void beforeMethod() {
  final StringBuilder builder = new StringBuilder();
  for (final String subExpression : subExpressions) {
    if (builder.length() > 0) {
      builder.append(" or ");
    }
    builder.append(subExpression);
  }
  final String expression = builder.toString();
  alarmExpression = new AlarmExpression(expression);
  alarmDefinition =
      new AlarmDefinition(tenantId, "Test CPU Alarm", "Description of Alarm",
          alarmExpression, "LOW", true, new ArrayList<String>());
  alarm = new Alarm(alarmDefinition, AlarmState.OK);
  subAlarms = new ArrayList<SubAlarm>(alarm.getSubAlarms());

  alarmEventForwarder = mock(AlarmEventForwarder.class);
  alarmDAO = mock(AlarmDAO.class);
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);
  bolt = new MockAlarmThreshholdBolt(alarmDAO, alarmDefinitionDAO, alarmEventForwarder);
  collector = mock(OutputCollector.class);
  final Map<String, String> config = new HashMap<>();
  final TopologyContext context = mock(TopologyContext.class);
  bolt.prepare(config, context, collector);

  for (SubAlarm subAlarm : subAlarms) {
    if (subAlarm.getExpression().getFunction().equals(AggregateFunction.LAST)) {
      lastSubAlarm = subAlarm;
    }
  }
  assertNotNull(lastSubAlarm, "Did not find a SubAlarm with Function of last");
  lastSubAlarm.setState(AlarmState.OK);
}
 

@SuppressWarnings("rawtypes")
@Override
public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) {
  this.collector = collector;
    /**
     * new NthLastModifiedTimeTracker(numSlots)
     * 新建一个追踪器，使用CircularFifoBuffer实现，循环缓冲。追踪数量为numSlots。追踪内容为time-since-last-modify。
     * 这个追踪器同样也是一个时间滑动窗口，并且和SlidingWindowCounter是一致的。
     */
  lastModifiedTracker = new NthLastModifiedTimeTracker(deriveNumWindowChunksFrom(this.windowLengthInSeconds,
      this.emitFrequencyInSeconds));
}
 

@Override
    public void prepare(Map conf, TopologyContext context, OutputCollector collector) {
        this.collector = collector;
        this.context = context;

        if (!initialized) {
            this.topologyId = (String) conf.get(StormConfigKey.TOPOLOGY_ID);
            this.datasource = (String) conf.get(StormConfigKey.DATASOURCE);
            this.zkRoot = Utils.buildZKTopologyPath(topologyId);
            try {
                this.zkconnect = (String) conf.get(StormConfigKey.ZKCONNECT);
                PropertiesHolder.initialize(this.zkconnect, zkRoot);
                GlobalCache.initialize(datasource);

//                String topic = PropertiesHolder.getProperties(Constants.Properties.CONFIGURE, Constants.ConfigureKey.DBUS_STATISTIC_TOPIC);

                if (producer != null) {
                    producer.close();
                }
                producer = createProducer();

                topicProvider = new DataOutputTopicProvider();
                reporter = AppenderMetricReporter.getInstance();
                //evictingQueue = IndexedEvictingQueue.create(SENT_QUEUE_SIZE);

                statSender = new KafkaStatSender();
                tableStatReporter = new TableMessageStatReporter(statSender);

                handlerManager = new BoltHandlerManager(buildProvider());

                initialized = true;
            } catch (Exception e) {
                logger.error(e.getMessage(), e);
                throw new InitializationException(e);
            }
        }
    }
 

@Override
public void doPrepare(Map conf, TopologyContext topologyContext,
        OutputCollector collector) throws IOException {
    super.doPrepare(conf, topologyContext, collector);
    protocolMDprefix = ConfUtils.getString(conf,
            ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, "");
    withRecordFormat(new WARCRecordFormat(protocolMDprefix));
    if (withRequestRecords) {
        addRecordFormat(new WARCRequestRecordFormat(protocolMDprefix), 0);
    }
}
 

/**
 * Verifies that the emitter does emit a {@code ProfileMeasurement}.
 *
 * @return The {@code ProfileMeasurement} that was emitted
 */
private ProfileMeasurement expectMeasurement(HBaseEmitter hbaseEmitter, OutputCollector collector) {

  ArgumentCaptor<Values> arg = ArgumentCaptor.forClass(Values.class);
  verify(collector, times(1)).emit(eq(hbaseEmitter.getStreamId()), arg.capture());
  Values values = arg.getValue();
  assertTrue(values.get(0) instanceof ProfileMeasurement);
  return (ProfileMeasurement) values.get(0);
}
 

@Test
public void testExecuteWithOutlinksLimit() throws IOException {
    stormConf.put("parser.emitOutlinks.max.per.page", 5);
    bolt.prepare(stormConf, TestUtil.getMockedTopologyContext(),
            new OutputCollector(output));

    parse("http://www.digitalpebble.com", "digitalpebble.com.html");

    List<List<Object>> statusTuples = output
            .getEmitted(Constants.StatusStreamName);

    // outlinks being limited by property
    Assert.assertEquals(5, statusTuples.size());
}