backtype.storm.tuple.Tuple Java Examples

The following examples show how to use backtype.storm.tuple.Tuple. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SequenceTestSplitRecord.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
    Long tupleId = input.getLong(0);
    Object object = input.getValue(1);

    if (object instanceof TradeCustomer) {

        TradeCustomer tradeCustomer = (TradeCustomer) object;

        Pair trade = tradeCustomer.getTrade();
        Pair customer = tradeCustomer.getCustomer();

        collector.emit(SequenceTopologyDef.TRADE_STREAM_ID, new Values(tupleId, trade));
        collector.emit(SequenceTopologyDef.CUSTOMER_STREAM_ID, new Values(tupleId, customer));
        emitCounter.update(2);
    }
}
 
Example #2
Source File: InOrderTestBolt.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple input, BasicOutputCollector collector) {
    Integer c1 = input.getInteger(0);
    Integer c2 = input.getInteger(1);

    Integer expect = expected.get(c1);
    if (expect == null)
        expect = 0;

    if (c2.intValue() == expect.intValue())
        successCounter.inc();
    else
        failCounter.inc();

    expect = c2 + 1;
    expected.put(c1, expect);
}
 
Example #3
Source File: AckTransactionBolt.java    From jstorm with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple input) {
    long rootId = getRootId(input);
    if (rootId != 0) {
        long batchId = ((TupleImplExt) input).getBatchId();
        String streamId = input.getSourceStreamId();
        Pair<Long, Integer> pendingBatch = batchXorTracker.getPendingBatch(batchId, streamId, true);
        if (pendingBatch == null) {
            pendingBatch = new Pair<>(0l, 0);
            batchXorTracker.putPendingBatch(batchId, streamId, pendingBatch);
        }
        pendingBatch.setFirst(JStormUtils.bit_xor(pendingBatch.getFirst(), rootId));
        int count = pendingBatch.getSecond();
        pendingBatch.setSecond(++count);
    }
    bolt.execute(input);
}
 
Example #4
Source File: FlattenJsonBoltTest.java    From cognition with Apache License 2.0 6 votes vote down vote up
@Test
public void testExecute(
    @Injectable Tuple input,
    @Injectable AbstractLogRecordBolt.RecordCollector collector,
    @Mocked LogRecord logRecord) {
  String tupleValue = "tuple-value";
  new Expectations(bolt) {{
    input.getValue(0);
    result = tupleValue.getBytes();
    new LogRecord(anyString);
    result = logRecord;
    bolt.parseJson(anyString, logRecord);
    collector.emit(logRecord);
  }};

  bolt.execute(input, collector);
}
 
Example #5
Source File: EntityStreamPersist.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple input) {
    List<? extends TaggedLogAPIEntity> entities = (List<? extends TaggedLogAPIEntity>) input.getValue(0);
    entityBucket.addAll(entities);

    if (entityBucket.size() < batchSize) {
        return;
    }

    try {
        GenericServiceAPIResponseEntity response = client.create(entityBucket);
        if (response.isSuccess()) {
            LOG.info("persist {} entities with starttime={}", entityBucket.size(), entityBucket.get(0).getTimestamp());
            collector.ack(input);
        } else {
            LOG.error("Service side error: {}", response.getException());
            collector.reportError(new IllegalStateException(response.getException()));
        }
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        collector.fail(input);
    }
    entityBucket.clear();
}
 
Example #6
Source File: IPZoneDataEnrichBolt.java    From eagle with Apache License 2.0 6 votes vote down vote up
@Override
public void executeWithEnrich(Tuple input, Map<String, IPZoneEntity> map) {
    try {
        Map<String, Object> toBeCopied = (Map<String, Object>) input.getValue(1);
        Map<String, Object> event = new TreeMap<String, Object>(toBeCopied); // shallow copy
        IPZoneEntity e = null;
        if (map != null) {
            e = map.get(event.get("host"));
        }
        event.put("securityZone", e == null ? "NA" : e.getSecurityZone());
        if (LOG.isDebugEnabled()) {
            LOG.debug("After IP zone lookup: " + event);
        }
        collector.emit(Arrays.asList(event.get("user"), event));
    } catch (Exception ex) {
        LOG.error("error joining data, ignore it", ex);
    } finally {
        collector.ack(input);
    }
}
 
Example #7
Source File: Throughput.java    From flink-perf with Apache License 2.0 6 votes vote down vote up
@Override
public void execute(Tuple input) {
	List<Object> vals = input.getValues();
	Long v = (Long) vals.get(0);
	v++;
	vals.set(0, v);
	if(withFt) {
		// anchor the output on the only input element (we pass through)
		collector.emit(input, vals);
		// acknowledge the element upstream.
		collector.ack(input);
	} else {
		// unanchored pass forward
		collector.emit(input.getValues());
	}
}
 
Example #8
Source File: MetricRegister.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void process(Object event) throws Exception {
    if (event instanceof Tuple) {
        registerMetrics((Tuple) event);
    } else if (event instanceof MetricsMetaBroadcastEvent) {
        broadcast();
    } else {
        zkCluster.report_task_error(context.getTopologyId(), context.getThisTaskId(),
                "Unknown event", ErrorConstants.WARN, ErrorConstants.CODE_USER);
        throw new RuntimeException("Unknown event");
    }
}
 
Example #9
Source File: KafkaBoltTest.java    From storm-kafka-0.8-plus with Apache License 2.0 5 votes vote down vote up
@Test
public void executeWithKey() throws Exception {
    String message = "value-123";
    String key = "key-123";
    Tuple tuple = generateTestTuple(key, message);
    bolt.execute(tuple);
    verify(collector).ack(tuple);
    verifyMessage(key, message);
}
 
Example #10
Source File: SlidingWindowTestAvgBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TupleWindow inputWindow) {
    int sum = 0;
    List<Tuple> tuplesInWindow = inputWindow.get();
    if (tuplesInWindow.size() > 0) {
        for (Tuple tuple : tuplesInWindow) {
            sum += (int) tuple.getValue(0);
        }
        asmCounter.update(sum / tuplesInWindow.size());
    }
}
 
Example #11
Source File: FastWordCountWindowTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple) {
    String sentence = tuple.getString(0);
    for (String word : sentence.split("\\s+")) {
        collector.emit(new Values(word));
    }
}
 
Example #12
Source File: TopologyDataPersistBolt.java    From eagle with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple input) {
    if (input == null) {
        return;
    }
    String serviceName = input.getStringByField(TopologyConstants.SERVICE_NAME_FIELD);
    TopologyEntityParserResult result = (TopologyEntityParserResult) input.getValueByField(TopologyConstants.TOPOLOGY_DATA_FIELD);
    Set<String> availableHostNames = new HashSet<String>();
    List<TopologyBaseAPIEntity> entitiesForDeletion = new ArrayList<>();
    List<TopologyBaseAPIEntity> entitiesToWrite = new ArrayList<>();

    filterEntitiesToWrite(result, availableHostNames, entitiesToWrite);

    String query = String.format("%s[@site=\"%s\"]{*}", serviceName, this.config.dataExtractorConfig.site);
    try {
        GenericServiceAPIResponseEntity<TopologyBaseAPIEntity> response = client.search().query(query).pageSize(Integer.MAX_VALUE).send();
        if (response.isSuccess() && response.getObj() != null) {
            for (TopologyBaseAPIEntity entity : response.getObj()) {
                if (!availableHostNames.isEmpty() && !availableHostNames.contains(generatePersistKey(entity))) {
                    entitiesForDeletion.add(entity);
                }
            }
        }
        deleteEntities(entitiesForDeletion, serviceName);
        writeEntities(entitiesToWrite, result.getMetrics(), serviceName);
        emitToKafkaBolt(result);
        this.collector.ack(input);
    } catch (Exception e) {
        LOG.error(e.getMessage(), e);
        this.collector.fail(input);
    }
}
 
Example #13
Source File: FastWordCountSessionEventTimeWindowTopology.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, Object state, TimeWindow window) {
    Map<String, Integer> counts = (Map<String, Integer>) state;
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null)
        count = 0;
    counts.put(word, ++count);
}
 
Example #14
Source File: WindowTestTotalRankingsBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
void updateRankingsWithTuple(Tuple tuple) {
    Rankings rankingsToBeMerged = (Rankings) tuple.getValue(0);
    super.getRankings().updateWith(rankingsToBeMerged);
    super.getRankings().pruneZeroCounts();

    LOG.info("TotalRankingsBolt updateRankingsWithTuple " + getRankings());
}
 
Example #15
Source File: TransactionFastWordCountWindowedState.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, IKvState<String, Integer> state, TimeWindow window) {
    String word = (String) tuple.getValue(0);
    Integer count = state.get(word);
    if (count == null)
        count = 0;
    state.put(word, ++count);
}
 
Example #16
Source File: ShellBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
private void handleAck(Object id) {
    Tuple acked = _inputs.remove(id);
    if (acked == null) {
        throw new RuntimeException("Acked a non-existent or already acked/failed id: " + id);
    }
    _collector.ack(acked);
}
 
Example #17
Source File: HdfsBolt.java    From storm-hdfs with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple) {
    try {
        byte[] bytes = this.format.format(tuple);
        synchronized (this.writeLock) {
            out.write(bytes);
            this.offset += bytes.length;

            if (this.syncPolicy.mark(tuple, this.offset)) {
                if (this.out instanceof HdfsDataOutputStream) {
                    ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(SyncFlag.UPDATE_LENGTH));
                } else {
                    this.out.hsync();
                }
                this.syncPolicy.reset();
            }
        }

        this.collector.ack(tuple);

        if(this.rotationPolicy.mark(tuple, this.offset)){
            rotateOutputFile(); // synchronized
            this.offset = 0;
            this.rotationPolicy.reset();
        }
    } catch (IOException e) {
        LOG.warn("write/sync failed.", e);
        this.collector.fail(tuple);
    }
}
 
Example #18
Source File: TransactionalWords.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple) {
    String key = tuple.getString(1);
    Integer curr = _counts.get(key);
    if (curr == null)
        curr = 0;
    _counts.put(key, curr + 1);
}
 
Example #19
Source File: MetricStreamPersistTest.java    From eagle with Apache License 2.0 5 votes vote down vote up
@Test
public void testStructuredMetricMapper() throws Exception {
    MetricDescriptor metricDefinition = MetricDescriptor
        .metricGroupByField("group")
        .siteAs("siteId")
        .namedByField("metric")
        .eventTimeByField("timestamp")
        .dimensionFields("host", "component", "site")
        .granularity(Calendar.MINUTE)
        .valueField("value");
    Config config = mock(Config.class);
    MetricStreamPersist metricStreamPersist = new MetricStreamPersist(metricDefinition, config);
    Field mapperField = metricStreamPersist.getClass().getDeclaredField("mapper");
    mapperField.setAccessible(true);

    Map event = new HashMap();
    event.put("timestamp", 1482106479564L);
    event.put("metric", "hadoop.memory.heapmemoryusage.used");
    event.put("component", "hbasemaster");
    event.put("site", "sandbox");
    event.put("value", 14460904.0);
    event.put("host", "xxx-xxx.int.xxx.com");

    Tuple tuple = Testing.testTuple(new Values("metric", event));
    MetricStreamPersist.MetricMapper mapper = (MetricStreamPersist.MetricMapper) mapperField.get(metricStreamPersist);

    GenericMetricEntity metricEntity = mapper.map(StreamConvertHelper.tupleToEvent(tuple).f1());

    Assert.assertEquals("prefix:hadoop.memory.heapmemoryusage.used, timestamp:1482106440000, humanReadableDate:2016-12-19 00:14:00,000, tags: component=hbasemaster,site=sandbox,host=xxx-xxx.int.xxx.com,, encodedRowkey:null", metricEntity.toString());
}
 
Example #20
Source File: WordCounterBolt.java    From storm-camel-example with Apache License 2.0 5 votes vote down vote up
@Override
public final void execute(final Tuple tuple, final BasicOutputCollector collector) {
    final String word = tuple.getString(0);
    int count = counts.getOrDefault(word, 0);
    count++;
    counts.put(word, count);

    collector.emit(new Values(word, count));
}
 
Example #21
Source File: WindowTestAbstractRankingBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
/**
 * This method functions as a template method (design pattern).
 */
@Override
public final void execute(Tuple tuple, BasicOutputCollector collector) {
    if (TupleUtils.isTick(tuple)) {
        getLogger().debug("Received tick tuple, triggering emit of current rankings");
        emitRankings(collector);
    } else {
        updateRankingsWithTuple(tuple);
    }
}
 
Example #22
Source File: PreprocessorBolt.java    From senti-storm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
  String text = tuple.getStringByField("text");
  List<String> tokens = (List<String>) tuple.getValueByField("tokens");

  // Preprocess
  List<String> preprocessedTokens = m_preprocessor.preprocess(tokens);

  if (m_logging) {
    LOG.info("Tweet: " + preprocessedTokens);
  }

  // Emit new tuples
  collector.emit(new Values(text, preprocessedTokens));
}
 
Example #23
Source File: WriterBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void execute(Tuple tuple) {
    // TODO Auto-generated method stub
    MetaTuple metaTuple = (MetaTuple)tuple.getValue(0);
    
    try {
        LOG.info("Messages:" + metaTuple);
        
    } catch (Exception e) {
        collector.fail(tuple);
        return ;
        //throw new FailedException(e);
    }
    
    collector.ack(tuple);
}
 
Example #24
Source File: TokenizerBolt.java    From senti-storm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple, BasicOutputCollector collector) {
  String text = tuple.getStringByField("text");

  List<String> tokens = Tokenizer.tokenize(text);

  if (m_logging) {
    LOG.info("Tweet: \"" + text + "\" Tokenized: " + tokens);
  }

  // Emit new tuples
  collector.emit(new Values(text, tokens));
}
 
Example #25
Source File: DefaultTupleMapper.java    From storm-trident-elasticsearch with Apache License 2.0 5 votes vote down vote up
/**
 * Returns a new {@link DefaultTupleMapper} that accept Byte[] as source field value.
 */
public static final DefaultTupleMapper newBinaryDefaultTupleMapper( ) {
    return new DefaultTupleMapper(new TupleMapper<String>() {
        @Override
        public String map(Tuple input) {
            try {
                return new String(input.getBinaryByField(FIELD_SOURCE), "UTF-8");
            } catch (UnsupportedEncodingException e) {
                throw new MappingException("Error while processing source as a byte[]", e);
            }
        }
    });
}
 
Example #26
Source File: TestBackpressure.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple) {
    String word = tuple.getString(0);
    Integer count = counts.get(word);
    if (count == null)
        count = 0;
    counts.put(word, ++count);
    if (isSlowdown == true) {
        JStormUtils.sleepMs(100);
    }
}
 
Example #27
Source File: EsQueryBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple tuple) {
  try {
    String index = mapper.getIndex(tuple);
    String type = mapper.getType(tuple);
    String id = mapper.getId(tuple);
    GetResponse response = client.prepareGet(index, type, id).execute()
        .actionGet();
    collector.emit(esOutputDeclarer.getValues(response.getSource()));
    collector.ack(tuple);
  } catch (Exception e) {
    collector.fail(tuple);
  }
}
 
Example #28
Source File: CoordinatedBolt.java    From jstorm with Apache License 2.0 5 votes vote down vote up
public void handlePostCommit(Tuple tuple) {
    basicCollector.setContext(tuple);
    try {
        BatchId id = (BatchId) tuple.getValue(0);
        ((IPostCommit) delegate).postCommit(id, basicCollector);

    } catch (Exception e) {
        LOG.info("Failed to do postCommit,", e);
    }
    collector.ack(tuple);
}
 
Example #29
Source File: TopologyMaster.java    From jstorm with Apache License 2.0 5 votes vote down vote up
@Override
public void execute(Tuple input) {
    TMHandler tmHandler = handlers.get(input.getSourceStreamId());
    if (tmHandler == null) {
        LOG.error("No handler of " + input.getSourceStreamId());
        tmContext.getCollector().fail(input);
        return;
    }

    TMEvent event = new TMEvent(tmHandler, input);
    threadPools.submit(event);
    tmContext.getCollector().ack(input);
}
 
Example #30
Source File: TotalWordCounter.java    From storm-hbase with Apache License 2.0 5 votes vote down vote up
public void execute(Tuple input, BasicOutputCollector collector) {
    total = total.add(new BigInteger(input.getValues().get(1).toString()));
    collector.emit(tuple(total.toString()));
    //prints the total with low probability.
    if(RANDOM.nextInt(1000) > 995) {
        LOG.info("Running total = " + total);
    }
}