Java Code Examples for storm.trident.tuple.TridentTuple

The following examples show how to use storm.trident.tuple.TridentTuple. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: jstorm   Source File: GroupedAggregator.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void aggregate(Object[] arr, TridentTuple tuple, TridentCollector collector) {
    GroupCollector groupColl = (GroupCollector) arr[0];
    Map<List, Object> val = (Map) arr[1];
    TridentTuple group = _groupFactory.create((TridentTupleView) tuple);
    TridentTuple input = _inputFactory.create((TridentTupleView) tuple);
    Object curr;
    if (!val.containsKey(group)) {
        curr = _agg.init(arr[2], groupColl);
        val.put((List) group, curr);
    } else {
        curr = val.get(group);
    }
    groupColl.currGroup = group;
    _agg.aggregate(curr, input, groupColl);

}
 
Example 2
Source Project: first-stories-twitter   Source File: TextProcessor.java    License: MIT License 6 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Status s = null;
	String tweetText = null;
	try {
		s = DataObjectFactory.createStatus((String) tuple.getValue(0));
		tweetText = tools.removeLinksAndReplies(tb.removeSpacesInBetween(s.getText()));
	} catch (Exception e) {
		LOG.error(e.toString());
	}

	Tweet t = null;
	if (s!=null)	//rarely Twitter4J can't parse the json to convert to Status and Status is null.
		t = new Tweet(s.getId(), tweetText);
	else
		t = new Tweet(-1, " ");
	
	collector.emit(new Values(t));
	
}
 
Example 3
Source Project: storm-example   Source File: GenerateBoards.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    GameState gameState = (GameState) tuple.get(0);
    Board currentBoard = gameState.getBoard();
    List<Board> history = new ArrayList<Board>();
    history.addAll(gameState.getHistory());
    history.add(currentBoard);

    if (!currentBoard.isEndState()) {
        String nextPlayer = Player.next(gameState.getPlayer());
        List<Board> boards = gameState.getBoard().nextBoards(nextPlayer);
        Log.debug("Generated [" + boards.size() + "] children boards for [" + gameState.toString() + "]");
        for (Board b : boards) {
            GameState newGameState = new GameState(b, history, nextPlayer);
            List<Object> values = new ArrayList<Object>();
            values.add(newGameState);
            collector.emit(values);
        }
    } else {
        Log.debug("End game found! [" + currentBoard + "]");
    }
}
 
Example 4
Source Project: storm-example   Source File: ScoreFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    GameState gameState = (GameState) tuple.get(0);
    String player = gameState.getPlayer();
    int score = gameState.score();

    List<Object> values = new ArrayList<Object>();
    values.add(gameState.getBoard());
    values.add(score);
    values.add(player);
    collector.emit(values);

    for (Board b : gameState.getHistory()) {
        player = Player.next(player);
        values = new ArrayList<Object>();
        values.add(b);
        values.add(score);
        values.add(player);
        collector.emit(values);
    }
}
 
Example 5
Source Project: storm-example   Source File: TweetSplitterFunction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String tweet = (String) tuple.getValue(0);
    LOG.error("SPLITTING TWEET [" + tweet + "]");
    Pattern p = Pattern.compile("[a-zA-Z]+");
    Matcher m = p.matcher(tweet);
    List<String> result = new ArrayList<String>();
    while (m.find()) {
        String word = m.group();
        if (word.length() > 0) {
            List<Object> newTuple = new ArrayList<Object>();
            newTuple.add(word);
            collector.emit(newTuple);
        }
    }
}
 
Example 6
Source Project: storm-example   Source File: StormFirehose.java    License: Apache License 2.0 6 votes vote down vote up
public synchronized void sendBatch(Long txId, List<TridentTuple> tuples) {
    BLOCKING_QUEUE = new ArrayBlockingQueue<TridentTuple>(tuples.size(), false, tuples);
    TRANSACTION_ID = txId;
    LOG.error("Beginning commit to Druid. [" + tuples.size() + "] messages, unlocking [START]");
    synchronized (START) {
        START.notify();
    }
    try {
        synchronized (FINISHED) {
            FINISHED.wait();
        }
    } catch (InterruptedException e) {
        LOG.error("Commit to Druid interrupted.");
    }
    LOG.info("Returning control to Storm.");
}
 
Example 7
Source Project: jstorm   Source File: PartitionPersistProcessor.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void finishBatch(ProcessorContext processorContext) {
    _collector.setContext(processorContext);
    Object batchId = processorContext.batchId;
    // since this processor type is a committer, this occurs in the commit phase
    List<TridentTuple> buffer = (List) processorContext.state[_context.getStateIndex()];

    // don't update unless there are tuples
    // this helps out with things like global partition persist, where multiple tasks may still
    // exist for this processor. Only want the global one to do anything
    // this is also a helpful optimization that state implementations don't need to manually do
    if (buffer.size() > 0) {
        Long txid = null;
        // this is to support things like persisting off of drpc stream, which is inherently unreliable
        // and won't have a tx attempt
        if (batchId instanceof TransactionAttempt) {
            txid = ((TransactionAttempt) batchId).getTransactionId();
        }
        _state.beginCommit(txid);
        _updater.updateState(_state, buffer, _collector);
        _state.commit(txid);
    }
}
 
Example 8
Source Project: storm-hdfs   Source File: HdfsState.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void execute(List<TridentTuple> tuples) throws IOException {
    boolean rotated = false;
    synchronized (this.writeLock) {
        for (TridentTuple tuple : tuples) {
            byte[] bytes = this.format.format(tuple);
            out.write(bytes);
            this.offset += bytes.length;

            if (this.rotationPolicy.mark(tuple, this.offset)) {
                rotateOutputFile();
                this.offset = 0;
                this.rotationPolicy.reset();
                rotated = true;
            }
        }
        if (!rotated) {
            if (this.out instanceof HdfsDataOutputStream) {
                ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
            } else {
                this.out.hsync();
            }
        }
    }
}
 
Example 9
Source Project: flink-perf   Source File: TridentWordCount.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
  String sentence = tuple.getString(0);
  for (String word : sentence.split(" ")) {
    collector.emit(new Values(word));
  }
}
 
Example 10
Source Project: trident-tutorial   Source File: Constants.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    for (T constant : constants) {
        collector.emit(new Values(constant));
    }

}
 
Example 11
Source Project: jstorm   Source File: CombinerAggStateUpdater.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void updateState(Snapshottable state, List<TridentTuple> tuples, TridentCollector collector) {
    if (tuples.size() != 1) {
        throw new IllegalArgumentException("Combiner state updater should receive a single tuple. Received: " + tuples.toString());
    }
    Object newVal = state.update(new CombinerValueUpdater(_agg, tuples.get(0).getValue(0)));
    collector.emit(new Values(newVal));
}
 
Example 12
Source Project: first-stories-twitter   Source File: VectorBuilder.java    License: MIT License 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Tweet tweet = (Tweet) tuple.getValue(0);
	String tweetBody = tweet.getBody();

       String words[] = tweetBody.toLowerCase().split(regex);
       if (words.length > 0) {
       	collector.emit(getValues(tweet, words));
       } else {
           tweetBody = "ONLYLINKSANDMENTIONZ";
           String dummyWord[] = {tweetBody};
           collector.emit(getValues(tweet, dummyWord));
       }
}
 
Example 13
Source Project: jstorm   Source File: ReturnResultsReducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(ReturnResultsState state, int streamIndex, TridentTuple input, TridentCollector collector) {
    if (streamIndex == 0) {
        state.returnInfo = input.getString(0);
    } else {
        state.results.add(input);
    }
}
 
Example 14
Source Project: flink-perf   Source File: TridentForwardThroughput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Matcher m = threeDigitAbbr.matcher(tuple.getString(0));
	if (m.matches()) {
		matches++;
	}

	if(start == 0) {
		start = System.currentTimeMillis();
	}
	received++;
	if(received % logfreq == 0) {
		long now = System.currentTimeMillis();


		// throughput for the last "logfreq" elements
		if(lastLog == -1) {
			// init (the first)
			lastLog = now;
			lastElements = received;
		} else {
			long timeDiff = now - lastLog;
			long elementDiff = received - lastElements;
			double ex = (1000/(double)timeDiff);
			LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, elementDiff*ex);
			// reinit
			lastLog = now;
			lastElements = received;
		}
	}

	if(tuple.getLong(1) != 0) {
		long lat = System.currentTimeMillis() - tuple.getLong(1);
		LOG.info("Latency {} ms from same machine", lat);
	}
}
 
Example 15
Source Project: trident-tutorial   Source File: Split.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(splitter==null){
        splitter = Splitter.on(on);
    }

    String string = tuple.getString(0);
    for (String spilt : splitter.split(string)) {
        collector.emit(new Values(spilt));
    }
}
 
Example 16
Source Project: trident-tutorial   Source File: ExtractLocation.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    Status status = (Status) tuple.get(0);
    Content content = (Content) tuple.get(1);

    collector.emit(new Values(status.getPlace().getCountryCode(), content.getContentName()));
}
 
Example 17
Source Project: jstorm   Source File: DefaultSequenceFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Writable key(TridentTuple tuple) {
    if(this.key == null){
        this.key  = new LongWritable();
    }
    this.key.set(tuple.getLongByField(this.keyField));
    return this.key;
}
 
Example 18
Source Project: jstorm   Source File: Equals.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean isKeep(TridentTuple tuple) {
    for (int i = 0; i < tuple.size() - 1; i++) {
        Object o1 = tuple.getValue(i);
        Object o2 = tuple.getValue(i + 1);
        if (o1 == null && o2 != null || o1 != null && !o1.equals(o2)) {
            return false;
        }
    }
    return true;
}
 
Example 19
Source Project: trident-tutorial   Source File: StringCounter.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void aggregate(Map<String, Integer> val, TridentTuple tuple, TridentCollector collector) {
    String loc = tuple.getString(0);
    Integer previousValue = val.get(loc);
    previousValue = previousValue == null ? 0 : previousValue;
    val.put(loc, previousValue + 1);
}
 
Example 20
Source Project: first-stories-twitter   Source File: BucketsStateQuery.java    License: MIT License 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, ArrayList<Tweet> collidingTweets,
		TridentCollector collector) {
	//emit by tweet id
	Tweet tw = (Tweet) tuple.getValue(0);
	collector.emit(new Values(tw.getID(), collidingTweets));
}
 
Example 21
Source Project: trident-tutorial   Source File: WithDefaultValue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(tuple.get(0) == null){
        collector.emit(new Values(t));
    }else{
        collector.emit(new Values(tuple.get(0)));
    }
}
 
Example 22
Source Project: storm-example   Source File: PrinterFunction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    FixMessageDto message = (FixMessageDto) tuple.getValue(0);
    Log.error("MESSAGE RECEIVED [" + message + "]");
    List<Object> values = new ArrayList<Object>();
    values.add(message);
    collector.emit(values);
}
 
Example 23
Source Project: flink-perf   Source File: TridentThroughput.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tridentTuple, TridentCollector tridentCollector) {
	Long id = tridentTuple.getLong(0);
	id++;
	Values v = new Values(id);
	tridentCollector.emit(v);
}
 
Example 24
Source Project: jstorm   Source File: StoreBasedTridentWindowManager.java    License: Apache License 2.0 5 votes vote down vote up
public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, List<String> keys) {
    if (tridentBatchTuple.tridentTuple != null) {
        return tridentBatchTuple.tridentTuple;
    }
    keys.add(tupleKey(tridentBatchTuple));
    return null;
}
 
Example 25
Source Project: jstorm   Source File: JoinerMultiReducer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, TridentCollector collector) {
    // TODO: do the inner join incrementally, emitting the cross join with this tuple, against all other sides
    // TODO: only do cross join if at least one tuple in each side
    List<List> side = state.sides[streamIndex];
    if (side.isEmpty()) {
        state.numSidesReceived++;
    }

    side.add(input);
    if (state.numSidesReceived == state.sides.length) {
        emitCrossJoin(state, collector, streamIndex, input);
    }
}
 
Example 26
Source Project: jstorm   Source File: CombinerAggregatorCombineImpl.java    License: Apache License 2.0 5 votes vote down vote up
public void aggregate(Result val, TridentTuple tuple, TridentCollector collector) {
    Object v = tuple.getValue(0);
    if (val.obj == null) {
        val.obj = v;
    } else {
        val.obj = _agg.combine(val.obj, v);
    }
}
 
Example 27
Source Project: jstorm   Source File: TridentReach.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    List l = (List) tuple.getValue(0);
    if (l != null) {
        for (Object o : l) {
            collector.emit(new Values(o));
        }
    }
}
 
Example 28
public void aggregateValue(TridentTuple tuple) {
    K key = mapper.getKey(tuple);
    V value = mapper.getValue(tuple);
    V currentValue = aggregateValues.get(key);
    V newValue;
    if (currentValue == null) {
        newValue = aggregator.init(tuple);
    } else {
        newValue = aggregator.combine(currentValue, value);
    }
    LOG.debug("Updating state [{}] ==> [{}]", new Object[]{key, newValue});
    aggregateValues.put(key, newValue);
}
 
Example 29
Source Project: jstorm   Source File: ProjectedProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void execute(ProcessorContext processorContext, String streamId, TridentTuple tuple) {
    TridentTuple toEmit = _factory.create(tuple);
    for (TupleReceiver r : _context.getReceivers()) {
        r.execute(processorContext, _context.getOutStreamId(), toEmit);
    }
}
 
Example 30
Source Project: jstorm   Source File: FilterNull.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean isKeep(TridentTuple tuple) {
    for (Object o : tuple) {
        if (o == null)
            return false;
    }
    return true;
}