storm.trident.tuple.TridentTuple Java Examples

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    GameState gameState = (GameState) tuple.get(0);
    Board currentBoard = gameState.getBoard();
    List<Board> history = new ArrayList<Board>();
    history.addAll(gameState.getHistory());
    history.add(currentBoard);

    if (!currentBoard.isEndState()) {
        String nextPlayer = Player.next(gameState.getPlayer());
        List<Board> boards = gameState.getBoard().nextBoards(nextPlayer);
        Log.debug("Generated [" + boards.size() + "] children boards for [" + gameState.toString() + "]");
        for (Board b : boards) {
            GameState newGameState = new GameState(b, history, nextPlayer);
            List<Object> values = new ArrayList<Object>();
            values.add(newGameState);
            collector.emit(values);
        }
    } else {
        Log.debug("End game found! [" + currentBoard + "]");
    }
}
 

@Override
public void aggregate(Object[] arr, TridentTuple tuple, TridentCollector collector) {
    GroupCollector groupColl = (GroupCollector) arr[0];
    Map<List, Object> val = (Map) arr[1];
    TridentTuple group = _groupFactory.create((TridentTupleView) tuple);
    TridentTuple input = _inputFactory.create((TridentTupleView) tuple);
    Object curr;
    if (!val.containsKey(group)) {
        curr = _agg.init(arr[2], groupColl);
        val.put((List) group, curr);
    } else {
        curr = val.get(group);
    }
    groupColl.currGroup = group;
    _agg.aggregate(curr, input, groupColl);

}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Status s = null;
	String tweetText = null;
	try {
		s = DataObjectFactory.createStatus((String) tuple.getValue(0));
		tweetText = tools.removeLinksAndReplies(tb.removeSpacesInBetween(s.getText()));
	} catch (Exception e) {
		LOG.error(e.toString());
	}

	Tweet t = null;
	if (s!=null)	//rarely Twitter4J can't parse the json to convert to Status and Status is null.
		t = new Tweet(s.getId(), tweetText);
	else
		t = new Tweet(-1, " ");
	
	collector.emit(new Values(t));
	
}
 

@Override
public void finishBatch(ProcessorContext processorContext) {
    _collector.setContext(processorContext);
    Object batchId = processorContext.batchId;
    // since this processor type is a committer, this occurs in the commit phase
    List<TridentTuple> buffer = (List) processorContext.state[_context.getStateIndex()];

    // don't update unless there are tuples
    // this helps out with things like global partition persist, where multiple tasks may still
    // exist for this processor. Only want the global one to do anything
    // this is also a helpful optimization that state implementations don't need to manually do
    if (buffer.size() > 0) {
        Long txid = null;
        // this is to support things like persisting off of drpc stream, which is inherently unreliable
        // and won't have a tx attempt
        if (batchId instanceof TransactionAttempt) {
            txid = ((TransactionAttempt) batchId).getTransactionId();
        }
        _state.beginCommit(txid);
        _updater.updateState(_state, buffer, _collector);
        _state.commit(txid);
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    GameState gameState = (GameState) tuple.get(0);
    String player = gameState.getPlayer();
    int score = gameState.score();

    List<Object> values = new ArrayList<Object>();
    values.add(gameState.getBoard());
    values.add(score);
    values.add(player);
    collector.emit(values);

    for (Board b : gameState.getHistory()) {
        player = Player.next(player);
        values = new ArrayList<Object>();
        values.add(b);
        values.add(score);
        values.add(player);
        collector.emit(values);
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    String tweet = (String) tuple.getValue(0);
    LOG.error("SPLITTING TWEET [" + tweet + "]");
    Pattern p = Pattern.compile("[a-zA-Z]+");
    Matcher m = p.matcher(tweet);
    List<String> result = new ArrayList<String>();
    while (m.find()) {
        String word = m.group();
        if (word.length() > 0) {
            List<Object> newTuple = new ArrayList<Object>();
            newTuple.add(word);
            collector.emit(newTuple);
        }
    }
}
 

@Override
public void execute(List<TridentTuple> tuples) throws IOException {
    boolean rotated = false;
    synchronized (this.writeLock) {
        for (TridentTuple tuple : tuples) {
            byte[] bytes = this.format.format(tuple);
            out.write(bytes);
            this.offset += bytes.length;

            if (this.rotationPolicy.mark(tuple, this.offset)) {
                rotateOutputFile();
                this.offset = 0;
                this.rotationPolicy.reset();
                rotated = true;
            }
        }
        if (!rotated) {
            if (this.out instanceof HdfsDataOutputStream) {
                ((HdfsDataOutputStream) this.out).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
            } else {
                this.out.hsync();
            }
        }
    }
}
 

public synchronized void sendBatch(Long txId, List<TridentTuple> tuples) {
    BLOCKING_QUEUE = new ArrayBlockingQueue<TridentTuple>(tuples.size(), false, tuples);
    TRANSACTION_ID = txId;
    LOG.error("Beginning commit to Druid. [" + tuples.size() + "] messages, unlocking [START]");
    synchronized (START) {
        START.notify();
    }
    try {
        synchronized (FINISHED) {
            FINISHED.wait();
        }
    } catch (InterruptedException e) {
        LOG.error("Commit to Druid interrupted.");
    }
    LOG.info("Returning control to Storm.");
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(tuple.get(0) == null){
        collector.emit(new Values(t));
    }else{
        collector.emit(new Values(tuple.get(0)));
    }
}
 

public TridentTuple collectTridentTupleOrKey(TridentBatchTuple tridentBatchTuple, List<String> keys) {
    if (tridentBatchTuple.tridentTuple != null) {
        return tridentBatchTuple.tridentTuple;
    }
    keys.add(tupleKey(tridentBatchTuple));
    return null;
}
 

@Override
public void execute(TridentTuple tuple, ArrayList<Tweet> collidingTweets,
		TridentCollector collector) {
	//emit by tweet id
	Tweet tw = (Tweet) tuple.getValue(0);
	collector.emit(new Values(tw.getID(), collidingTweets));
}
 

@Override
public void aggregate(Map<String, Integer> val, TridentTuple tuple, TridentCollector collector) {
    String loc = tuple.getString(0);
    Integer previousValue = val.get(loc);
    previousValue = previousValue == null ? 0 : previousValue;
    val.put(loc, previousValue + 1);
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Matcher m = threeDigitAbbr.matcher(tuple.getString(0));
	if (m.matches()) {
		matches++;
	}

	if(start == 0) {
		start = System.currentTimeMillis();
	}
	received++;
	if(received % logfreq == 0) {
		long now = System.currentTimeMillis();


		// throughput for the last "logfreq" elements
		if(lastLog == -1) {
			// init (the first)
			lastLog = now;
			lastElements = received;
		} else {
			long timeDiff = now - lastLog;
			long elementDiff = received - lastElements;
			double ex = (1000/(double)timeDiff);
			LOG.info("During the last {} ms, we received {} elements. That's {} elements/second/core", timeDiff, elementDiff, elementDiff*ex);
			// reinit
			lastLog = now;
			lastElements = received;
		}
	}

	if(tuple.getLong(1) != 0) {
		long lat = System.currentTimeMillis() - tuple.getLong(1);
		LOG.info("Latency {} ms from same machine", lat);
	}
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
	Tweet tweet = (Tweet) tuple.getValue(0);
	String tweetBody = tweet.getBody();

       String words[] = tweetBody.toLowerCase().split(regex);
       if (words.length > 0) {
       	collector.emit(getValues(tweet, words));
       } else {
           tweetBody = "ONLYLINKSANDMENTIONZ";
           String dummyWord[] = {tweetBody};
           collector.emit(getValues(tweet, dummyWord));
       }
}
 

@Override
public Writable key(TridentTuple tuple) {
    if(this.key == null){
        this.key  = new LongWritable();
    }
    this.key.set(tuple.getLongByField(this.keyField));
    return this.key;
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(splitter==null){
        splitter = Splitter.on(on);
    }

    String string = tuple.getString(0);
    for (String spilt : splitter.split(string)) {
        collector.emit(new Values(spilt));
    }
}
 

@Override
public void execute(ReturnResultsState state, int streamIndex, TridentTuple input, TridentCollector collector) {
    if (streamIndex == 0) {
        state.returnInfo = input.getString(0);
    } else {
        state.results.add(input);
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    FixMessageDto message = (FixMessageDto) tuple.getValue(0);
    Log.error("MESSAGE RECEIVED [" + message + "]");
    List<Object> values = new ArrayList<Object>();
    values.add(message);
    collector.emit(values);
}
 

@Override
public void execute(TridentTuple tridentTuple, TridentCollector tridentCollector) {
	Long id = tridentTuple.getLong(0);
	id++;
	Values v = new Values(id);
	tridentCollector.emit(v);
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    for (T constant : constants) {
        collector.emit(new Values(constant));
    }

}
 

@Override
public void execute(JoinState state, int streamIndex, TridentTuple group, TridentTuple input, TridentCollector collector) {
    // TODO: do the inner join incrementally, emitting the cross join with this tuple, against all other sides
    // TODO: only do cross join if at least one tuple in each side
    List<List> side = state.sides[streamIndex];
    if (side.isEmpty()) {
        state.numSidesReceived++;
    }

    side.add(input);
    if (state.numSidesReceived == state.sides.length) {
        emitCrossJoin(state, collector, streamIndex, input);
    }
}
 

public void aggregate(Result val, TridentTuple tuple, TridentCollector collector) {
    Object v = tuple.getValue(0);
    if (val.obj == null) {
        val.obj = v;
    } else {
        val.obj = _agg.combine(val.obj, v);
    }
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    List l = (List) tuple.getValue(0);
    if (l != null) {
        for (Object o : l) {
            collector.emit(new Values(o));
        }
    }
}
 

public void aggregateValue(TridentTuple tuple) {
    K key = mapper.getKey(tuple);
    V value = mapper.getValue(tuple);
    V currentValue = aggregateValues.get(key);
    V newValue;
    if (currentValue == null) {
        newValue = aggregator.init(tuple);
    } else {
        newValue = aggregator.combine(currentValue, value);
    }
    LOG.debug("Updating state [{}] ==> [{}]", new Object[]{key, newValue});
    aggregateValues.put(key, newValue);
}
 

@Override
public void execute(ProcessorContext processorContext, String streamId, TridentTuple tuple) {
    TridentTuple toEmit = _factory.create(tuple);
    for (TupleReceiver r : _context.getReceivers()) {
        r.execute(processorContext, _context.getOutStreamId(), toEmit);
    }
}
 

@Override
public boolean isKeep(TridentTuple tuple) {
    for (Object o : tuple) {
        if (o == null)
            return false;
    }
    return true;
}
 

@Override
public void execute(TridentTuple tuple, TridentCollector collector) {
    if(extracter == null) extracter = new ContentExtracter();

    String rawTweetJson = (String)tuple.get(0);
    Status parsed = parse(rawTweetJson);
    User user = parsed.getUser();

    for (Content content : extracter.extract(parsed)) {
        collector.emit(new Values(parsed, content, user));
    }
}
 

@Override
public void complete(Map<TridentTuple, Object> state, TridentCollector collector) {
    for (Map.Entry e : state.entrySet()) {
        TridentTuple group = (TridentTuple) e.getKey();
        Object val = e.getValue();
        _reducer.complete(val, group, collector);
    }
}
 

@Override
public byte[] format(TridentTuple tuple) {
    StringBuilder sb = new StringBuilder();
    int size = this.fields.size();
    for(int i = 0; i < size; i++){
        sb.append(tuple.getValueByField(fields.get(i)));
        if(i != size - 1){
            sb.append(this.fieldDelimiter);
        }
    }
    sb.append(this.recordDelimiter);
    return sb.toString().getBytes();
}
 

protected T valueFromTuple(TridentTuple tuple) {
    // when there is no input field then the whole tuple is considered for comparison.
    Object value = null;
    if (inputFieldName != null && tuple != null) {
        value =  tuple.getValueByField(inputFieldName);
    } else {
        value = tuple;
    }

    log.debug("value from tuple is [{}] with input field [{}] and tuple [{}]", value, inputFieldName, tuple);

    return (T) value;
}