org.apache.storm.spout.SpoutOutputCollector Java Examples

@Test
public void testFailedConsumer() {
    PulsarSpoutConfiguration pulsarSpoutConf = new PulsarSpoutConfiguration();
    pulsarSpoutConf.setServiceUrl(serviceUrl);
    pulsarSpoutConf.setTopic("persistent://invalidTopic");
    pulsarSpoutConf.setSubscriptionName(subscriptionName);
    pulsarSpoutConf.setMessageToValuesMapper(messageToValuesMapper);
    pulsarSpoutConf.setFailedRetriesTimeout(1, TimeUnit.SECONDS);
    pulsarSpoutConf.setMaxFailedRetries(2);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    pulsarSpoutConf.setMetricsTimeIntervalInSecs(60);
    pulsarSpoutConf.setSubscriptionType(SubscriptionType.Shared);
    PulsarSpout spout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("new-test" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    try {
        spout.open(Maps.newHashMap(), context, collector);
        fail("should have failed as consumer creation failed");
    } catch (IllegalStateException e) {
        // Ok
    }
}
 

@Override
@SuppressWarnings("rawtypes")
public void open(Map conf, TopologyContext context,
		SpoutOutputCollector collector) {
	
	_collector = collector;
	try {
		Reader =  new FileReader();
		jsons = Reader.readFromFile(_filename);

	} catch (Throwable e)
	{
		System.out.println("Could not read sample JSONs");
		e.printStackTrace();
	}
	
}
 

/**
 * 初始化collectors
 */
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {

    this.collector = collector;
    this.zkConnect = (String) conf.get(FullPullConstants.ZKCONNECT);
    this.topologyId = (String) conf.get(FullPullConstants.FULL_PULLER_TOPOLOGY_ID);
    this.dsName = (String) conf.get(FullPullConstants.DS_NAME);
    this.shardsProcessManager = new ShardsProcessManager();
    try {
        //设置topo类型，用于获取配置信息路径
        FullPullHelper.setTopologyType(FullPullConstants.FULL_PULLER_TYPE);
        loadRunningConf(null);
        FullPullHelper.updatePendingTasksToHistoryTable(dsName, FullPullConstants.FULLPULL_PENDING_TASKS_OP_PULL_TOPOLOGY_RESTART, consumer,
                commonProps.getProperty(FullPullConstants.FULL_PULL_MEDIANT_TOPIC));
    } catch (Exception e) {
        logger.error(e.getMessage(), e);
        throw new InitializationException();
    }
    logger.info("[pull spout] {} init complete!", topologyId);
}
 

@Override
public void open(Map stormConf, TopologyContext topologyContext, SpoutOutputCollector outputCollector) {
	logger.trace("{} Storm init", this);
	collector = outputCollector;
	super.init(stormConf, topologyContext);

	try {
		if (ackSignature != null) {
			ackMethod = ackSignature.findMethod(beanType);
			logger.info("{} uses {} for transaction acknowledgement",
					this, ackMethod.toGenericString());
		}
		if (failSignature != null) {
			failMethod = failSignature.findMethod(beanType);
			logger.info("{} uses {} for transaction failures",
					this, failMethod.toGenericString());
		}
	} catch (ReflectiveOperationException e) {
		throw new IllegalStateException("Unusable transaction signature", e);
	}
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void open(Map conf, TopologyContext context,
        SpoutOutputCollector collector) {
    _collector = collector;
    record = Optional.empty();

    maxContentSize = ConfUtils.getInt(conf, "http.content.limit", -1);
    if (contentBufferSize > maxContentSize) {
        // no need to buffer more content than max. used
        contentBufferSize = maxContentSize;
    }
    storeHTTPHeaders = ConfUtils.getBoolean(conf, "http.store.headers",
            false);
    protocolMDprefix = ConfUtils.getString(conf,
            ProtocolResponse.PROTOCOL_MD_PREFIX_PARAM, protocolMDprefix);

    int metricsTimeBucketSecs = ConfUtils.getInt(conf,
            "fetcher.metrics.time.bucket.secs", 10);
    eventCounter = context.registerMetric("warc_spout_counter",
            new MultiCountMetric(), metricsTimeBucketSecs);
}
 

@Override
public void open(@SuppressWarnings("rawtypes") Map conf,
        TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;

    // check that there is only one instance of it
    int totalTasks = context
            .getComponentTasks(context.getThisComponentId()).size();
    if (totalTasks > 1) {
        throw new RuntimeException(
                "Can't have more than one instance of the MemorySpout");
    }

    Date now = new Date();
    for (String u : startingURLs) {
        LOG.debug("About to deserialize {} ", u);
        List<Object> tuple = scheme.deserialize(ByteBuffer.wrap(u
                .getBytes(StandardCharsets.UTF_8)));
        add((String) tuple.get(0), (Metadata) tuple.get(1), now);
    }
    context.registerMetric("queue_size", () -> queue.size(), 10);
}
 

@Override
@SuppressWarnings("rawtypes")
public void open(Map map, TopologyContext topologyContext,
                 SpoutOutputCollector spoutOutputCollector) {
  RandomString randomString = new RandomString(WORD_LENGTH);
  for (int i = 0; i < ARRAY_LENGTH; i++) {
    StringBuilder sb = new StringBuilder(randomString.nextString());
    for (int j = 1; j < SENTENCE_LENGTH; j++) {
      sb.append(" ");
      sb.append(randomString.nextString());
    }
    sentences[i] = sb.toString();
  }

  collector = spoutOutputCollector;
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
  logger.info("Opened");
  this.collector = collector;
  logger.info(" topic = " + kafkaSpoutConfig.kafkaConsumerConfiguration.getTopic());
  this.spoutName = String.format("%s-%d", context.getThisComponentId(), context.getThisTaskId());

  Properties kafkaProperties =
      KafkaConsumerProperties.createKafkaProperties(kafkaSpoutConfig.kafkaConsumerConfiguration);
  // Have to use a different consumer.id for each spout so use the storm taskId. Otherwise,
  // zookeeper complains about a conflicted ephemeral node when there is more than one spout
  // reading from a topic
  kafkaProperties.setProperty("consumer.id", String.valueOf(context.getThisTaskId()));
  ConsumerConfig consumerConfig = new ConsumerConfig(kafkaProperties);
  this.consumerConnector = Consumer.createJavaConsumerConnector(consumerConfig);
}
 

@Override
protected void setup() throws Exception {
    super.internalSetup();
    super.producerBaseSetup();

    serviceUrl = pulsar.getWebServiceAddress();

    pulsarSpoutConf = new PulsarSpoutConfiguration();
    pulsarSpoutConf.setServiceUrl(serviceUrl);
    pulsarSpoutConf.setTopic(topic);
    pulsarSpoutConf.setSubscriptionName(subscriptionName);
    pulsarSpoutConf.setMessageToValuesMapper(messageToValuesMapper);
    pulsarSpoutConf.setFailedRetriesTimeout(1, TimeUnit.SECONDS);
    pulsarSpoutConf.setMaxFailedRetries(2);
    pulsarSpoutConf.setSharedConsumerEnabled(true);
    pulsarSpoutConf.setMetricsTimeIntervalInSecs(60);
    pulsarSpoutConf.setSubscriptionType(SubscriptionType.Shared);
    spout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    mockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(mockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(0);
    spout.open(Maps.newHashMap(), context, collector);
    producer = pulsarClient.newProducer().topic(topic).create();
}
 

@Test
public void testSharedConsumer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);

    otherSpout.close();

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}
 

@Test
public void testNoSharedConsumer() throws Exception {
    TopicStats topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
    pulsarSpoutConf.setSharedConsumerEnabled(false);
    PulsarSpout otherSpout = new PulsarSpout(pulsarSpoutConf, PulsarClient.builder());
    MockSpoutOutputCollector otherMockCollector = new MockSpoutOutputCollector();
    SpoutOutputCollector collector = new SpoutOutputCollector(otherMockCollector);
    TopologyContext context = mock(TopologyContext.class);
    when(context.getThisComponentId()).thenReturn("test-spout-" + methodName);
    when(context.getThisTaskId()).thenReturn(1);
    otherSpout.open(Maps.newHashMap(), context, collector);

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 2);

    otherSpout.close();

    topicStats = admin.topics().getStats(topic);
    assertEquals(topicStats.subscriptions.get(subscriptionName).consumers.size(), 1);
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = new InterceptingSpoutOutputCollector(collector);

    if (spout != null) {
        spout.open(conf, context, this.collector);
    }
    log.info("Opened TestSpout");
}
 

@SuppressWarnings("rawtypes")
public void open(
    Map conf,
    TopologyContext context,
    SpoutOutputCollector aCollector) {
  collector = aCollector;
  words = new String[]{"Africa", "Europe", "Asia", "America", "Antarctica", "Australia"};
  rand = new Random();
}
 

@Override
protected void processMessage(byte[] message, SpoutOutputCollector collector) {
  List<List<?>> events = deserializer.deserialize(message);
  if (events != null) {
    for (final List<?> event : events) {
      final Object eventToSend = event.get(0);
      if (!(eventToSend instanceof Serializable)) {
        logger.error("Class {} is not Serializable: {}", eventToSend.getClass(), eventToSend);
        continue;
      }
      collector.emit(new Values(eventToSend));
    }
  }
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.conf = conf;
    this.collector = collector;
    this.zkServers = (String) conf.get(Constants.ZOOKEEPER_SERVERS);
    this.extractorName = (String) conf.get(Constants.EXTRACTOR_TOPOLOGY_ID);
    this.extractorRoot = Constants.EXTRACTOR_ROOT + "/";

    reloadConfig(null);
    ackOrRollbackStartTime = System.currentTimeMillis();
    checkSurvivalStartTime = System.currentTimeMillis();

    logger.info("CanalClientSpout() open finished");
}
 

@Override
public void prepare(Config cfg, TaskContext context) {
  LOG.info("Preparing storm-spout : " + this.id);

  this.stormSpout.open(
      cfg.toMap(),
      new TopologyContext(context),
      new SpoutOutputCollector(
          this.id,
          context,
          this.outFieldsForEdge,
          this.keyedOutEdges
      )
  );
}
 

@Override
@SuppressWarnings("HiddenField")
public void open(Map conf, TopologyContext context, SpoutOutputCollector
    collector) {
  this.collector = collector;
  this.rand = new Random();
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1);
    isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false);
}
 

@Override
    public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
        initialized = false;
        this.context = context;

        // Spout internals
        this.collector = collector;
        numUncommittedOffsets = 0;

        // Offset management
        firstPollOffsetStrategy = kafkaSpoutConfig.getFirstPollOffsetStrategy();
        // with AutoCommitMode, offset will be periodically committed in the background by Kafka consumer
//        consumerAutoCommitMode = kafkaSpoutConfig.isConsumerAutoCommitMode();
        //永远设置为false
        consumerAutoCommitMode = false;

        // Retries management
        retryService = kafkaSpoutConfig.getRetryService();

        if (!consumerAutoCommitMode) {     // If it is auto commit, no need to commit offsets manually
            commitTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getOffsetsCommitPeriodMs(), TimeUnit.MILLISECONDS);
        }
        refreshSubscriptionTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS);

        acked = new HashMap<>();
        emitted = new HashSet<>();
        waitingToEmit = Collections.emptyListIterator();

        LOG.info("Kafka Spout opened with the following configuration: {}", kafkaSpoutConfig);
    }
 

@SuppressWarnings("rawtypes")
public void open(
    Map conf,
    TopologyContext context,
    SpoutOutputCollector acollector) {
  collector = acollector;
  words = new String[]{"nathan", "mike", "jackson", "golda", "bertels"};
  rand = new Random();
}
 

@SuppressWarnings("rawtypes")
public void open(
    Map conf,
    TopologyContext context,
    SpoutOutputCollector acollector) {
  collector = acollector;
  words = new String[]{"nathan", "mike", "jackson", "golda", "bertels"};
  rand = new Random();
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    // Add the Storm Config and the context as is, in case any PubSubs need it.
    config.set(BulletStormConfig.STORM_CONFIG, conf);
    config.set(BulletStormConfig.STORM_CONTEXT, context);

    this.collector = collector;
}
 

@Override
protected List<Integer> emit(List<Object> tuple,
                             Message message,
                             SpoutOutputCollector spoutOutputCollector) {
  String stream = streamSplitter.selectStream(tuple, message);
  return spoutOutputCollector.emit(stream, tuple, getDeliveryTag(message));
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    try {
        connector.initialize();
    } catch (BulletDSLException e) {
        throw new RuntimeException("Could not open DSLSpout.", e);
    }
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    tick = 0;
    lastTickTime = System.currentTimeMillis();
    id = new Random().nextInt();
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = new InterceptingSpoutOutputCollector(collector);

    if (spout != null) {
        spout.open(conf, context, this.collector);
    }
    log.info("Opened TestSpout");
}
 

@Override
public void open(Map conf, final TopologyContext context, final SpoutOutputCollector spoutOutputCollector) {

    Preconditions.checkNotNull(this.spoutConfig.getZkBinLogStateConfig(),
            "Zookeeper Config cannot be null");

    Preconditions.checkNotNull(this.spoutConfig.getMysqlConfig(),
            "Mysql Config cannot be null");

    LOGGER.info("Initiating MySql Spout with config {}", this.spoutConfig.toString());

    this.collector          = spoutOutputCollector;
    this.topologyInstanceId = context.getStormId();
    this.topologyName       = conf.get(Config.TOPOLOGY_NAME).toString();
    this.databaseName       = this.spoutConfig.getMysqlConfig().getDatabase();
    this.sidelineStrategy   = this.spoutConfig.getFailureConfig().getSidelineStrategy();
    this.sidelineStrategy.initialize(conf, context);

    initializeAndRegisterAllMetrics(context, this.spoutConfig.getMetricsTimeBucketSizeInSecs());

    txQueue = this.clientFactory.initializeBuffer(this.spoutConfig.getBufferCapacity());
    zkClient = this.clientFactory.getZkClient(conf, this.spoutConfig.getZkBinLogStateConfig());
    mySqlClient = this.clientFactory.getMySqlClient(this.spoutConfig.getMysqlConfig());
    openReplicatorClient = this.clientFactory.getReplicatorClient(mySqlClient, zkClient);

    begin();
}
 

@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
    this.spoutOutputCollector = spoutOutputCollector;
    this.queue = new LinkedBlockingQueue<>(1000);

    this.spoutReceiver = new NiFiSpoutReceiver();
    this.spoutReceiver.setDaemon(true);
    this.spoutReceiver.setName("NiFi Spout Receiver");
    this.spoutReceiver.start();
}
 

@Override
public void open(Map map, TopologyContext topologyContext, SpoutOutputCollector spoutOutputCollector) {
    this.spoutOutputCollector = spoutOutputCollector;
    this.queue = new LinkedBlockingQueue<>(1000);

    this.spoutReceiver = new NiFiSpoutReceiver();
    this.spoutReceiver.setDaemon(true);
    this.spoutReceiver.setName("NiFi Spout Receiver");
    this.spoutReceiver.start();
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void open(Map conf, TopologyContext context,
        SpoutOutputCollector collector) {

    super.open(conf, context, collector);

    maxDocsPerBucket = ConfUtils.getInt(conf,
            Constants.SQL_MAX_DOCS_BUCKET_PARAM_NAME, 5);

    tableName = ConfUtils.getString(conf,
            Constants.SQL_STATUS_TABLE_PARAM_NAME, "urls");

    maxNumResults = ConfUtils.getInt(conf,
            Constants.SQL_MAXRESULTS_PARAM_NAME, 100);

    try {
        connection = SQLUtil.getConnection(conf);
    } catch (SQLException ex) {
        LOG.error(ex.getMessage(), ex);
        throw new RuntimeException(ex);
    }

    // determine bucket this spout instance will be in charge of
    int totalTasks = context
            .getComponentTasks(context.getThisComponentId()).size();
    if (totalTasks > 1) {
        logIdprefix = "[" + context.getThisComponentId() + " #"
                + context.getThisTaskIndex() + "] ";
        bucketNum = context.getThisTaskIndex();
    }
}