org.apache.storm.topology.BasicOutputCollector Java Examples

The following examples show how to use org.apache.storm.topology.BasicOutputCollector. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlobStoreAPIWordCountTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    // Thread Polling every 5 seconds to update the wordSet seconds which is
    // used in FilterWords bolt to filter the words
    try {
        if (!poll) {
            wordSet = parseFile(fileName);
            pollTime = System.currentTimeMillis();
            poll = true;
        } else {
            if ((System.currentTimeMillis() - pollTime) > 5000) {
                wordSet = parseFile(fileName);
                pollTime = System.currentTimeMillis();
            }
        }
    } catch (IOException exp) {
        throw new RuntimeException(exp);
    }
    if (wordSet != null && !wordSet.contains(word)) {
        collector.emit(new Values(word));
    }
}
 
Example #2
Source File: WordCountTopologyNode.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null) {
        count = 0;
    }
    count++;
    counts.put(word, count);
    collector.emit(new Values(word, count));
}
 
Example #3
Source File: ConvertIPBolt.java    From storm-statistic with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
    byte[] binary = input.getBinary(0);
    String line = new String(binary);
    String[] fields = line.split("\t");

    if(fields == null || fields.length < 10) {
        return;
    }

    // 获取ip和mid
    String ip = fields[1];
    String mid = fields[2];

    // 根据ip获取其所属地(省份)
    String province = null;
    if (ip != null) {
        Jedis jedis = JedisUtil.getJedis();
        province = jedis.hget("ip_info_en", ip);
        // 需要释放jedis的资源,否则会报can not get resource from the pool
        JedisUtil.returnJedis(jedis);
    }

    // 发送数据到下一个bolt,只发送实现业务功能需要的province和mid
    collector.emit(new Values(province, mid));

}
 
Example #4
Source File: ReachTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    Object id = tuple.getValue(0);
    String url = tuple.getString(1);
    List<String> tweeters = TWEETERS_DB.get(url);
    if (tweeters != null) {
        for (String tweeter : tweeters) {
            collector.emit(new Values(id, tweeter));
        }
    }
}
 
Example #5
Source File: ReachTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    Object id = tuple.getValue(0);
    String tweeter = tuple.getString(1);
    List<String> followers = FOLLOWERS_DB.get(tweeter);
    if (followers != null) {
        for (String follower : followers) {
            collector.emit(new Values(id, follower));
        }
    }
}
 
Example #6
Source File: AnchoredWordCount.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split("\\s+")) {
        collector.emit(new Values(word, 1));
    }
}
 
Example #7
Source File: AnchoredWordCount.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null) {
        count = 0;
    }
    count++;
    counts.put(word, count);
    collector.emit(new Values(word, count));
}
 
Example #8
Source File: FastWordCountTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split("\\s+")) {
        collector.emit(new Values(word, 1));
    }
}
 
Example #9
Source File: FastWordCountTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null) {
        count = 0;
    }
    count++;
    counts.put(word, count);
    collector.emit(new Values(word, count));
}
 
Example #10
Source File: AbstractRankerBolt.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
/**
 * This method functions as a template method (design pattern).
 */
@Override
public final void execute(Tuple tuple, BasicOutputCollector collector) {
    if (TupleUtils.isTick(tuple)) {
        getLogger().debug("Received tick tuple, triggering emit of current rankings");
        emitRankings(collector);
    } else {
        updateRankingsWithTuple(tuple);
    }
}
 
Example #11
Source File: InOrderDeliveryTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    Integer c1 = tuple.getInteger(0);
    Integer c2 = tuple.getInteger(1);
    Integer exp = expected.get(c1);
    if (exp == null) exp = 0;
    if (c2.intValue() != exp.intValue()) {
        System.out.println(c1 + " " + c2 + " != " + exp);
        throw new FailedException(c1 + " " + c2 + " != " + exp);
    }
    exp = c2 + 1;
    expected.put(c1, exp);
}
 
Example #12
Source File: WordCountTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null) {
        count = 0;
    }
    count++;
    counts.put(word, count);
    collector.emit(new Values(word, count));
}
 
Example #13
Source File: TotalRankingsBoltTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldEmitSomethingIfTickTupleIsReceived() {
    // given
    Tuple tickTuple = MockTupleHelpers.mockTickTuple();
    BasicOutputCollector collector = mock(BasicOutputCollector.class);
    TotalRankingsBolt bolt = new TotalRankingsBolt();

    // when
    bolt.execute(tickTuple, collector);

    // then
    // verifyZeroInteractions(collector);
    verify(collector).emit(any(Values.class));
}
 
Example #14
Source File: TotalRankingsBoltTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldEmitNothingIfNormalTupleIsReceived() {
    // given
    Tuple normalTuple = mockRankingsTuple(ANY_OBJECT, ANY_COUNT);
    BasicOutputCollector collector = mock(BasicOutputCollector.class);
    TotalRankingsBolt bolt = new TotalRankingsBolt();

    // when
    bolt.execute(normalTuple, collector);

    // then
    verifyZeroInteractions(collector);
}
 
Example #15
Source File: IntermediateRankingsBoltTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldEmitSomethingIfTickTupleIsReceived() {
    // given
    Tuple tickTuple = MockTupleHelpers.mockTickTuple();
    BasicOutputCollector collector = mock(BasicOutputCollector.class);
    IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();

    // when
    bolt.execute(tickTuple, collector);

    // then
    // verifyZeroInteractions(collector);
    verify(collector).emit(any(Values.class));
}
 
Example #16
Source File: IntermediateRankingsBoltTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldEmitNothingIfNormalTupleIsReceived() {
    // given
    Tuple normalTuple = mockRankableTuple(ANY_OBJECT, ANY_COUNT);
    BasicOutputCollector collector = mock(BasicOutputCollector.class);
    IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();

    // when
    bolt.execute(normalTuple, collector);

    // then
    verifyZeroInteractions(collector);
}
 
Example #17
Source File: ABolt.java    From chuidiang-ejemplos with GNU Lesser General Public License v3.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
   long now = tuple.getLong(0);
   long offset = now-lastNow;
   lastNow = now;
   int value = tuple.getInteger(1);
   sum += value;
   collector.emit(new Values(offset,sum));
   log.info("data results emitted");
}
 
Example #18
Source File: DataPrinter.java    From chuidiang-ejemplos with GNU Lesser General Public License v3.0 5 votes vote down vote up
@Override
public void execute(Tuple value, BasicOutputCollector arg1) {
   for (String field:value.getFields()){
      log.info(MessageFormat.format("{0} = {1}",field, value.getValueByField(field).toString()));
   }
   
}
 
Example #19
Source File: FilteringBolt.java    From tutorials with MIT License 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
    int operation = tuple.getIntegerByField("operation");
    if(operation > 0 ) {
        basicOutputCollector.emit(tuple.getValues());
    }
}
 
Example #20
Source File: ParserBoltTest.java    From logparser with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    Fields fields = tuple.getFields();
    Assert.assertEquals(1, fields.size());
    TestRecord record = (TestRecord) tuple.getValueByField(OUTPUT_FIELD_NAME);
    record.assertIsValid();
    LOG.info("Test passed");
}
 
Example #21
Source File: HttpdLoglineParserBolt.java    From logparser with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
    String apacheLogLine = tuple.getStringByField(readFieldName);
    try {
        List<Object> out = new ArrayList<>();
        out.add(parser.parse(apacheLogLine));
        collector.emit(out);
    } catch (MissingDissectorsException
            |InvalidDissectorException
            |DissectionFailure e) {
        e.printStackTrace();
    }
}
 
Example #22
Source File: KeyedScottyWindowOperator.java    From scotty-window-processor with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector basicOutputCollector) {
    //The input is a tuple
    Key currentKey = (Key) tuple.getValue(0);
    if (!slicingWindowOperatorMap.containsKey(currentKey)) {
        slicingWindowOperatorMap.put(currentKey, initWindowOperator());
    }
    SlicingWindowOperator<Value> slicingWindowOperator = slicingWindowOperatorMap.get(currentKey);
    //We only process the Value of a tuple
    slicingWindowOperator.processElement((Value) tuple.getValue(1), tuple.getLong(2));
    processWatermark(currentKey, tuple.getLong(2), basicOutputCollector);
}
 
Example #23
Source File: PartitionDataBolt.java    From incubator-retired-pirk with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector outputCollector)
{
  int hash = tuple.getIntegerByField(StormConstants.HASH_FIELD);
  json = (JSONObject) tuple.getValueByField(StormConstants.JSON_DATA_FIELD);

  try
  {
    List<BigInteger> partitions = QueryUtils.partitionDataElement(qSchema, json, embedSelector);

    logger.debug("HashSelectorsAndPartitionDataBolt processing {} outputting results - {}", json.toString(), partitions.size());

    // splitPartitions determines whether each partition piece is sent individually or the full Array is sent together.
    // Since processing in the follow-on bolt (EncRowCalcBolt) is computationally expensive, current working theory is
    // that splitting them up allows for better throughput. Though maybe with better knowledge/tuning of Storm internals
    // and paramters (e.g. certain buffer sizes), it may make no difference.
    if (splitPartitions)
    {
      for (BigInteger partition : partitions)
      {
        outputCollector.emit(new Values(hash, partition));
      }
    }
    else
    {
      outputCollector.emit(new Values(hash, partitions));
    }

  } catch (Exception e)
  {
    logger.warn("Failed to partition data for record -- " + json + "\n", e);
  }
}
 
Example #24
Source File: TestIBasicPrintBolt.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
  System.out.println("The configuration method has set \"someProperty\" to : "
      + this.someProperty);
  System.out.println("The configuration method has set TestUnits to " + testUnits);
  System.out.println("Emitting : " + input);
  collector.emit(tuple(input.getValues().get(0)));

}
 
Example #25
Source File: KeyedScottyWindowOperator.java    From scotty-window-processor with Apache License 2.0 5 votes vote down vote up
private void processWatermark(Key currentKey, long timeStamp, BasicOutputCollector basicOutputCollector) {
    // Every tuple represents a Watermark with its timestamp.
    // A watermark is processed if it is greater than the old watermark, i.e. monotonically increasing.
    // We process watermarks every watermarkEvictionPeriod in event-time
    if (timeStamp > lastWatermark + watermarkEvictionPeriod) {
        for (SlicingWindowOperator<Value> slicingWindowOperator : this.slicingWindowOperatorMap.values()) {
            List<AggregateWindow> aggregates = slicingWindowOperator.processWatermark(timeStamp);
            for (AggregateWindow<Value> aggregateWindow : aggregates) {
                basicOutputCollector.emit(new Values(currentKey, aggregateWindow));
            }
        }
        lastWatermark = timeStamp;
    }
}
 
Example #26
Source File: SentenceWordCountTopology.java    From twister2 with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
  String sentence = tuple.getString(0);
  System.out.println("Sentence received : " + sentence);
  for (String word : sentence.split("\\s+")) {
    collector.emit(new Values(word, 1));
  }
}
 
Example #27
Source File: TestNameCounter.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
public void execute(Tuple input, BasicOutputCollector collector) {
  String word = getTupleValue(input, 0);
  int count = 0;
  if (counts.containsKey(word)) {
    count = counts.get(word);
  }
  count++;
  counts.put(word, count);
  collector.emit(tuple(word, count));
}
 
Example #28
Source File: SentenceWordCountTopology.java    From twister2 with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
  String word = tuple.getString(0);
  Integer count = counts.get(word);
  if (count == null) {
    count = 0;
  }
  count++;
  counts.put(word, count);
  System.out.println(word + ":" + count);
}
 
Example #29
Source File: SplitSentenceBolt.java    From storm_spring_boot_demo with MIT License 5 votes vote down vote up
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
    String sentence = input.getStringByField("value");
    for(int i=0;i<sentence.length();i++){
        char word = sentence.charAt(i);
        collector.emit(new Values(word+""));//分词emit,只有1个单词的元组Tuple
    }
}
 
Example #30
Source File: AbstractRankerBolt.java    From storm_spring_boot_demo with MIT License 5 votes vote down vote up
/**
 * This method functions as a template method (design pattern).
 */
@Override
public final void execute(Tuple tuple, BasicOutputCollector collector) {
  if (TupleUtils.isTick(tuple)) {
    getLogger().debug("Received tick tuple, triggering emit of current rankings");
    emitRankings(collector);
  }
  else {
    updateRankingsWithTuple(tuple);
  }
}