Java Code Examples for backtype.storm.utils.Utils.DEFAULT_STREAM_ID

@Test
public void testExecuteUniqueMultipleStreams() {
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	IncSpout spout = new IncSpout(streamIds);
	
	TestSpoutOutputCollector collector = new TestSpoutOutputCollector();
	spout.open(null, null, new SpoutOutputCollector(collector));
	
	List<List<Object>> result = new LinkedList<List<Object>>();
	
	for(int i = 0; i < 5; ++i) {
		ArrayList<Object> attributes = new ArrayList<Object>();
		attributes.add(new Long(i));
		result.add(attributes);
		
		spout.nextTuple();
	}
	
	for(String stream : streamIds) {
		Assert.assertEquals(result, collector.output.get(stream));
	}
}
 

@Test
public void testExecuteAllEqualsMultipleStreams() {
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	IncSpout spout = new IncSpout(streamIds, 1.0, 1);
	
	TestSpoutOutputCollector collector = new TestSpoutOutputCollector();
	spout.open(null, null, new SpoutOutputCollector(collector));
	
	List<List<Object>> result = new LinkedList<List<Object>>();
	
	for(int i = 0; i < 5; ++i) {
		ArrayList<Object> attributes = new ArrayList<Object>();
		attributes.add(new Long(0));
		result.add(attributes);
		
		spout.nextTuple();
	}
	
	for(String stream : streamIds) {
		Assert.assertEquals(result, collector.output.get(stream));
	}
}
 

@Test
public void testExecuteMultipleStreams() {
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	IncSpout spout = new IncSpout(streamIds, this.r.nextDouble(), 1);
	
	TestSpoutOutputCollector collector = new TestSpoutOutputCollector();
	spout.open(null, null, new SpoutOutputCollector(collector));
	
	for(int i = 0; i < 50; ++i) {
		spout.nextTuple();
	}
	
	for(String stream : streamIds) {
		List<Object> first = collector.output.get(stream).removeFirst();
		for(List<Object> second : collector.output.get(stream)) {
			Assert.assertTrue(((Long)first.get(0)).longValue() <= ((Long)second.get(0)).longValue());
			first = second;
		}
	}
}
 

@Test
public void testExecuteMultipleStreams() {
	int numberOfAttributes = 1 + this.r.nextInt(10);
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	RandomSpout spout = new RandomSpout(numberOfAttributes, 100, streamIds);
	
	TestSpoutOutputCollector collector = new TestSpoutOutputCollector();
	spout.open(null, null, new SpoutOutputCollector(collector));
	
	for(int i = 0; i < 50; ++i) {
		spout.nextTuple();
		for(String stream : streamIds) {
			Assert.assertTrue(collector.output.get(stream).size() == i + 1); // size of result
			Assert.assertTrue(collector.output.get(stream).get(i).size() == numberOfAttributes);
			for(int j = 0; j < numberOfAttributes; ++j) {
				Assert.assertTrue(0 < ((Integer)collector.output.get(stream).get(i).get(j)).intValue());
				Assert.assertTrue(100 >= ((Integer)collector.output.get(stream).get(i).get(j)).intValue());
			}
		}
	}
	
}
 

private void handleEmit(Map action) throws InterruptedException {
	String stream = (String) action.get("stream");
	if (stream == null)
		stream = Utils.DEFAULT_STREAM_ID;
	Long task = (Long) action.get("task");
	List<Object> tuple = (List) action.get("tuple");
	List<Tuple> anchors = new ArrayList<Tuple>();
	Object anchorObj = action.get("anchors");
	if (anchorObj != null) {
		if (anchorObj instanceof String) {
			anchorObj = Arrays.asList(anchorObj);
		}
		for (Object o : (List) anchorObj) {
			Tuple t = _inputs.get((String) o);
			if (t == null) {
				throw new RuntimeException("Anchored onto " + o
						+ " after ack/fail");
			}
			anchors.add(t);
		}
	}
	if (task == null) {
		List<Integer> outtasks = _collector.emit(stream, anchors, tuple);
		Object need_task_ids = action.get("need_task_ids");
		if (need_task_ids == null
				|| ((Boolean) need_task_ids).booleanValue()) {
			_pendingWrites.put(outtasks);
		}
	} else {
		_collector.emitDirect((int) task.longValue(), stream, anchors,
				tuple);
	}
}
 

@Test
public void testConfigureStreamGrouping_default(
    @Injectable String prevComponent,
    @Injectable Configuration boltConf,
    @Injectable BoltDeclarer declarer) throws Exception {
  new Expectations(topology) {{
    boltConf.getString(STREAM_GROUPING_CONF_TYPE, STREAM_GROUPING_LOCAL_OR_SHUFFLE);
    result = STREAM_GROUPING_SHUFFLE;
    boltConf.getString(STREAM_ID, Utils.DEFAULT_STREAM_ID);
    result = Utils.DEFAULT_STREAM_ID;
    declarer.shuffleGrouping(prevComponent, Utils.DEFAULT_STREAM_ID);
  }};

  topology.configureStreamGrouping(prevComponent, boltConf, declarer);
}
 

@Test
public void testConfigureStreamGrouping_localOrShuffleGrouping(
    @Injectable String prevComponent,
    @Injectable Configuration boltConf,
    @Injectable BoltDeclarer declarer) throws Exception {
  new Expectations(topology) {{
    boltConf.getString(STREAM_GROUPING_CONF_TYPE, STREAM_GROUPING_LOCAL_OR_SHUFFLE);
    result = STREAM_GROUPING_LOCAL_OR_SHUFFLE;
    boltConf.getString(STREAM_ID, Utils.DEFAULT_STREAM_ID);
    result = Utils.DEFAULT_STREAM_ID;
    declarer.localOrShuffleGrouping(prevComponent, Utils.DEFAULT_STREAM_ID);
  }};

  topology.configureStreamGrouping(prevComponent, boltConf, declarer);
}
 

@Test(timeout = 1000)
public void testNextTupleNoEmit() {
	final int batchSize = 6 + this.r.nextInt(5);
	final int skipInterval = 2 + this.r.nextInt(4);
	
	final String streamId = Utils.DEFAULT_STREAM_ID;
	final String sourceId = "sourceId";
	
	
	Map<String, Grouping> receiver = new HashMap<String, Grouping>();
	receiver.put("receiverId", mock(Grouping.class));
	
	HashMap<String, Map<String, Grouping>> targets = new HashMap<String, Map<String, Grouping>>();
	targets.put(streamId, receiver);
	
	TopologyContext context = mock(TopologyContext.class);
	when(context.getThisComponentId()).thenReturn(sourceId);
	when(context.getThisTargets()).thenReturn(targets);
	when(context.getComponentOutputFields(sourceId, streamId)).thenReturn(new Fields("dummy"));
	
	IncSpout userSpout = new IncSpout(skipInterval);
	SpoutOutputBatcher batcher = new SpoutOutputBatcher(userSpout, batchSize);
	
	TestSpoutOutputCollector collector = new TestSpoutOutputCollector();
	batcher.open(null, context, new SpoutOutputCollector(collector));
	
	int iterations = (int)Math.ceil(batchSize / (skipInterval - 1.));
	for(int i = 1; i < iterations; ++i) {
		batcher.nextTuple();
		Assert.assertNull(collector.output.get(Utils.DEFAULT_STREAM_ID));
	}
	batcher.nextTuple();
	
	Assert.assertEquals(1, collector.output.get(Utils.DEFAULT_STREAM_ID).size());
}
 

@Test
public void testExecuteMultipleStreams() {
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	ForwardBolt bolt = new ForwardBolt(new Fields("dummy"), streamIds);
	
	TestOutputCollector collector = new TestOutputCollector();
	bolt.prepare(null, null, new OutputCollector(collector));
	
	LinkedList<Tuple> tuples = new LinkedList<Tuple>();
	List<List<Object>> result = new LinkedList<List<Object>>();
	
	for(int i = 0; i < 3; ++i) {
		ArrayList<Object> attributes = new ArrayList<Object>();
		attributes.add(new Integer(i));
		
		tuples.add(mock(Tuple.class));
		when(tuples.get(i).getValues()).thenReturn(attributes);
		result.add(attributes);
		
		bolt.execute(tuples.get(i));
		Assert.assertEquals(tuples, collector.acked);
	}
	
	for(String stream : streamIds) {
		Assert.assertEquals(result, collector.output.get(stream));
	}
}
 

private void querySubprocess(Object query) {
	try {
		_process.writeMessage(query);

		while (true) {
			Map action = _process.readMessage();
			String command = (String) action.get("command");
			if (command.equals("sync")) {
				return;
			} else if (command.equals("log")) {
				String msg = (String) action.get("msg");
				LOG.info("Shell msg: " + msg);
			} else if (command.equals("emit")) {
				String stream = (String) action.get("stream");
				if (stream == null)
					stream = Utils.DEFAULT_STREAM_ID;
				Long task = (Long) action.get("task");
				List<Object> tuple = (List) action.get("tuple");
				Object messageId = (Object) action.get("id");
				if (task == null) {
					List<Integer> outtasks = _collector.emit(stream, tuple,
							messageId);
					Object need_task_ids = action.get("need_task_ids");
					if (need_task_ids == null
							|| ((Boolean) need_task_ids).booleanValue()) {
						_process.writeMessage(outtasks);
					}
				} else {
					_collector.emitDirect((int) task.longValue(), stream,
							tuple, messageId);
				}
			}
		}
	} catch (IOException e) {
		throw new RuntimeException(e);
	}
}
 

@Test
public void testDeclareOutputFieldsMultipleStreams() {
	Fields schema = new Fields("dummy");
	OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
	
	String[] streamIds = new String[] {Utils.DEFAULT_STREAM_ID, "myStreamId"};
	ForwardBolt bolt = new ForwardBolt(schema, streamIds);
	bolt.declareOutputFields(declarer);
	
	for(String stream : streamIds) {
		verify(declarer).declareStream(stream, schema);
	}
	verifyNoMoreInteractions(declarer);
}
 

@Override
public ShellMsg readShellMsg() throws IOException, NoOutputException {
       JSONObject msg = (JSONObject) readMessage();
       ShellMsg shellMsg = new ShellMsg();

       String command = (String) msg.get("command");
       shellMsg.setCommand(command);

       Object id = msg.get("id");
       shellMsg.setId(id);

       String log = (String) msg.get("msg");
       shellMsg.setMsg(log);

       String stream = (String) msg.get("stream");
       if (stream == null)
           stream = Utils.DEFAULT_STREAM_ID;
       shellMsg.setStream(stream);

       Object taskObj = msg.get("task");
       if (taskObj != null) {
           shellMsg.setTask((Long) taskObj);
       } else {
           shellMsg.setTask(0);
       }

       Object need_task_ids = msg.get("need_task_ids");
       if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) {
           shellMsg.setNeedTaskIds(true);
       } else {
           shellMsg.setNeedTaskIds(false);
       }

       shellMsg.setTuple((List) msg.get("tuple"));

       //List<Tuple> anchors = new ArrayList<Tuple>();
       Object anchorObj = msg.get("anchors");
       if (anchorObj != null) {
           if (anchorObj instanceof String) {
               anchorObj = Arrays.asList(anchorObj);
           }
           for (Object o : (List) anchorObj) {
               shellMsg.addAnchor((String) o);
           }
       }
      
       Object nameObj = msg.get("name"); 
       String metricName = null;
       if (nameObj != null && nameObj instanceof String) {
           metricName = (String) nameObj;
       }
       shellMsg.setMetricName(metricName);
       
       Object paramsObj = msg.get("params");
       shellMsg.setMetricParams(paramsObj);

       if (command.equals("log")) {
           Object logLevelObj = msg.get("level");
           if (logLevelObj != null && logLevelObj instanceof Long) {
               long logLevel = (Long)logLevelObj;
               shellMsg.setLogLevel((int)logLevel);
           }
       }

       return shellMsg;
   }
 

public ShellMsg readShellMsg() throws IOException, NoOutputException {
    JSONObject msg = (JSONObject) readMessage();
    ShellMsg shellMsg = new ShellMsg();

    String command = (String) msg.get("command");
    shellMsg.setCommand(command);

    Object id = msg.get("id");
    shellMsg.setId(id);

    String log = (String) msg.get("msg");
    shellMsg.setMsg(log);

    String stream = (String) msg.get("stream");
    if (stream == null)
        stream = Utils.DEFAULT_STREAM_ID;
    shellMsg.setStream(stream);

    Object taskObj = msg.get("task");
    if (taskObj != null) {
        shellMsg.setTask((Long) taskObj);
    } else {
        shellMsg.setTask(0);
    }

    Object need_task_ids = msg.get("need_task_ids");
    if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) {
        shellMsg.setNeedTaskIds(true);
    } else {
        shellMsg.setNeedTaskIds(false);
    }

    shellMsg.setTuple((List) msg.get("tuple"));

    //List<Tuple> anchors = new ArrayList<Tuple>();
    Object anchorObj = msg.get("anchors");
    if (anchorObj != null) {
        if (anchorObj instanceof String) {
            anchorObj = Arrays.asList(anchorObj);
        }
        for (Object o : (List) anchorObj) {
            shellMsg.addAnchor((String) o);
        }
    }
   
    Object nameObj = msg.get("name"); 
    String metricName = null;
    if (nameObj != null && nameObj instanceof String) {
        metricName = (String) nameObj;
    }
    shellMsg.setMetricName(metricName);
    
    Object paramsObj = msg.get("params");
    shellMsg.setMetricParams(paramsObj);

    if (command.equals("log")) {
        Object logLevelObj = msg.get("level");
        if (logLevelObj != null && logLevelObj instanceof Long) {
            long logLevel = (Long)logLevelObj;
            shellMsg.setLogLevel((int)logLevel);
        }
    }

    return shellMsg;
}
 

public ShellMsg readShellMsg() throws IOException, NoOutputException {
    JSONObject msg = (JSONObject) readMessage();
    ShellMsg shellMsg = new ShellMsg();

    String command = (String) msg.get("command");
    shellMsg.setCommand(command);

    Object id = msg.get("id");
    shellMsg.setId(id);

    String log = (String) msg.get("msg");
    shellMsg.setMsg(log);

    String stream = (String) msg.get("stream");
    if (stream == null)
        stream = Utils.DEFAULT_STREAM_ID;
    shellMsg.setStream(stream);

    Object taskObj = msg.get("task");
    if (taskObj != null) {
        shellMsg.setTask((Long) taskObj);
    } else {
        shellMsg.setTask(0);
    }

    Object need_task_ids = msg.get("need_task_ids");
    if (need_task_ids == null || ((Boolean) need_task_ids).booleanValue()) {
        shellMsg.setNeedTaskIds(true);
    } else {
        shellMsg.setNeedTaskIds(false);
    }

    shellMsg.setTuple((List) msg.get("tuple"));

    List<Tuple> anchors = new ArrayList<Tuple>();
    Object anchorObj = msg.get("anchors");
    if (anchorObj != null) {
        if (anchorObj instanceof String) {
            anchorObj = Arrays.asList(anchorObj);
        }
        for (Object o : (List) anchorObj) {
            shellMsg.addAnchor((String) o);
        }
    }

    Object nameObj = msg.get("name");
    String metricName = null;
    if (nameObj != null && nameObj instanceof String) {
        metricName = (String) nameObj;
    }
    shellMsg.setMetricName(metricName);

    Object paramsObj = msg.get("params");
    shellMsg.setMetricParams(paramsObj);

    if (command.equals("log")) {
        Object logLevelObj = msg.get("level");
        if (logLevelObj != null && logLevelObj instanceof Long) {
            long logLevel = (Long) logLevelObj;
            shellMsg.setLogLevel((int) logLevel);
        }
    }

    return shellMsg;
}
 

public FixedTuple(List<Object> values) {
	this.stream = Utils.DEFAULT_STREAM_ID;
	this.values = values;
}
 

/**
 * Instantiates a new {@link IncSpout} that emits unique values with step size one to the default output stream.
 */
public IncSpout() {
	this(new String[] {Utils.DEFAULT_STREAM_ID}, 0.0, 1, 0, System.currentTimeMillis());
}
 

/**
 * Instantiates a new {@link IncSpout} that emits unique values with step size one to the default output stream.
 * Each {@code skipInterval} {@link #nextTuple()} call will not emit a tuple. If {@code skipInterval} is smaller
 * than 2, skipping is disabled.
 * 
 * @param skipInterval
 *            The interval between two {@link #nextTuple()} calls that do not emit.
 */
public IncSpout(int skipInterval) {
	this(new String[] {Utils.DEFAULT_STREAM_ID}, 0.0, 1, skipInterval, System.currentTimeMillis());
}
 

/**
 * Instantiates a new {@link RandomSpout} that emits tuples with a single attribute to the default output stream.
 * 
 * @param maxValue
 *            The maximum value of the generated numbers (inclusive).
 */
public RandomSpout(int maxValue) {
	this(1, maxValue, new String[] {Utils.DEFAULT_STREAM_ID}, System.currentTimeMillis());
}
 

/**
 * Instantiates a new {@link RandomSpout} that emits tuples with {@code numberOfAttributes} attributes to the
 * default output stream.
 * 
 * @param numberOfAttributes
 *            The number of attributes of the output tuples.
 * @param maxValue
 *            The maximum value of the generated numbers (inclusive).
 */
public RandomSpout(int numberOfAttributes, int maxValue) {
	this(numberOfAttributes, maxValue, new String[] {Utils.DEFAULT_STREAM_ID}, System.currentTimeMillis());
}
 

/**
 * Instantiates a new {@link ForwardBolt} for the given tuple schema that emits all tuples to the default output
 * stream.
 * 
 * @param schema
 *            The schema of the input (and output) tuples.
 */
public ForwardBolt(Fields schema) {
	this(schema, new String[] {Utils.DEFAULT_STREAM_ID});
}