org.apache.storm.generated.SpoutSpec Java Examples

/**
 * Gets the Thrift object representing the topology.
 *
 * @return the Thrift definition representing the topology
 */
@SuppressWarnings("deprecation")
public StormTopology getRawTopology() {

  StormTopology stormTopology = new StormTopology();
  Map<String, SpoutSpec> spouts = new HashMap<>();
  for (TopologyAPI.Spout spout : this.delegate.getRawTopology().getSpoutsList()) {
    spouts.put(spout.getComp().getName(), new SpoutSpec(spout));
  }

  Map<String, Bolt> bolts = new HashMap<>();
  for (TopologyAPI.Bolt bolt : this.delegate.getRawTopology().getBoltsList()) {
    bolts.put(bolt.getComp().getName(), new Bolt(bolt));
  }

  stormTopology.set_spouts(spouts);
  stormTopology.set_bolts(bolts);
  return stormTopology;
}
 

private void addTopologyInputs(Referenceable topologyReferenceable,
                               Map<String, SpoutSpec> spouts,
                               Map stormConf,
                               String topologyOwner, List<Referenceable> dependentEntities) throws IllegalAccessException {
    final ArrayList<Referenceable> inputDataSets = new ArrayList<>();
    for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
        Serializable instance = Utils.javaDeserialize(
                entry.getValue().get_spout_object().get_serialized_java(), Serializable.class);

        String simpleName = instance.getClass().getSimpleName();
        final Referenceable datasetRef = createDataSet(simpleName, topologyOwner, instance, stormConf, dependentEntities);
        if (datasetRef != null) {
            inputDataSets.add(datasetRef);
        }
    }

    topologyReferenceable.set("inputs", inputDataSets);
}
 

private void addTopologyInputs(Map<String, SpoutSpec> spouts, Map stormConf, String topologyOwner, AtlasEntity topology, AtlasEntityExtInfo entityExtInfo) {
    List<AtlasEntity> inputs = new ArrayList<>();

    for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
        Serializable instance = Utils.javaDeserialize(entry.getValue().get_spout_object().get_serialized_java(), Serializable.class);
        String       dsType   = instance.getClass().getSimpleName();
        AtlasEntity  dsEntity = addDataSet(dsType, topologyOwner, instance, stormConf, entityExtInfo);

        if (dsEntity != null) {
            inputs.add(dsEntity);
        }
    }

    topology.setRelationshipAttribute("inputs", AtlasTypeUtil.getAtlasRelatedObjectIds(inputs, RELATIONSHIP_DATASET_PROCESS_INPUTS));
}
 

private List<AtlasEntity> createTopologyGraph(StormTopology stormTopology, Map<String, SpoutSpec> spouts, Map<String, Bolt> bolts) {
    // Add graph of nodes in the topology
    Map<String, AtlasEntity> nodeEntities = new HashMap<>();

    addSpouts(spouts, nodeEntities);
    addBolts(bolts, nodeEntities);

    addGraphConnections(stormTopology, nodeEntities);

    return new ArrayList<>(nodeEntities.values());
}
 

private void addSpouts(Map<String, SpoutSpec> spouts, Map<String, AtlasEntity> nodeEntities) {
    for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
        String      spoutName = entry.getKey();
        AtlasEntity spout     = createSpoutInstance(spoutName, entry.getValue());

        nodeEntities.put(spoutName, spout);
    }
}
 

private AtlasEntity createSpoutInstance(String spoutName, SpoutSpec stormSpout) {
    AtlasEntity         spout         = new AtlasEntity(StormDataTypes.STORM_SPOUT.getName());
    Serializable        instance      = Utils.javaDeserialize(stormSpout.get_spout_object().get_serialized_java(), Serializable.class);
    Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);

    spout.setAttribute(AtlasClient.NAME, spoutName);
    spout.setAttribute("driverClass", instance.getClass().getName());
    spout.setAttribute("conf", flatConfigMap);

    return spout;
}
 

private ArrayList<Referenceable> createTopologyGraph(StormTopology stormTopology,
                                                     Map<String, SpoutSpec> spouts,
                                                     Map<String, Bolt> bolts) throws Exception {
    // Add graph of nodes in the topology
    final Map<String, Referenceable> nodeEntities = new HashMap<>();
    addSpouts(spouts, nodeEntities);
    addBolts(bolts, nodeEntities);

    addGraphConnections(stormTopology, nodeEntities);

    ArrayList<Referenceable> nodes = new ArrayList<>();
    nodes.addAll(nodeEntities.values());
    return nodes;
}
 

private void addSpouts(Map<String, SpoutSpec> spouts,
                       Map<String, Referenceable> nodeEntities) throws IllegalAccessException {
    for (Map.Entry<String, SpoutSpec> entry : spouts.entrySet()) {
        final String spoutName = entry.getKey();
        Referenceable spoutReferenceable = createSpoutInstance(
                spoutName, entry.getValue());
        nodeEntities.put(spoutName, spoutReferenceable);
    }
}
 

private Referenceable createSpoutInstance(String spoutName,
                                          SpoutSpec stormSpout) throws IllegalAccessException {
    Referenceable spoutReferenceable = new Referenceable(StormDataTypes.STORM_SPOUT.getName());
    spoutReferenceable.set(AtlasClient.NAME, spoutName);

    Serializable instance = Utils.javaDeserialize(
            stormSpout.get_spout_object().get_serialized_java(), Serializable.class);
    spoutReferenceable.set("driverClass", instance.getClass().getName());

    Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);
    spoutReferenceable.set("conf", flatConfigMap);

    return spoutReferenceable;
}
 

@Test
public void build() throws Exception {
	beansXml = "<breeze:topology id='t1'>" +
			"<breeze:spout id='s1' beanType='eu.icolumbo.breeze.TestBean' signature='ping()' outputFields='feed'/>" +
			"<breeze:bolt id='b1' beanType='eu.icolumbo.breeze.TestBean' signature='echo(feed)' outputFields='replay' scatterOutput='true'/>" +
			"<breeze:bolt beanType='eu.icolumbo.breeze.TestBean' signature='drain(replay)' parallelism='2'/>" +
			"</breeze:topology>";
	refresh();

	StormTopology topology = getBean("t1", StormTopology.class);
	assertEquals("spout count", 1, topology.get_spouts_size());
	assertEquals("bolt count", 2, topology.get_bolts_size());

	SpringSpout spout = getBean("s1", SpringSpout.class);
	assertEquals("spout ID", "s1", spout.getId());
	assertEquals("spout scatter", false, spout.getScatterOutput());
	SpringBolt bolt = getBean("b1", SpringBolt.class);
	assertEquals("bolt ID", "b1", bolt.getId());
	assertEquals("bolt scatter", true, bolt.getScatterOutput());

	Map<String, SpoutSpec> topologySpouts = topology.get_spouts();
	SpoutSpec spoutSpec = topologySpouts.get("s1");
	assertNotNull("s1 spec", spoutSpec);

	Map<String, Bolt> topologyBolts = topology.get_bolts();
	Bolt boltSpec = topologyBolts.get("b1");
	assertNotNull("b1 spec", boltSpec);

	String anonymousBoltId = null;
	for (String id : topologyBolts.keySet())
		if (! "b1".equals(id))
			anonymousBoltId = id;
	assertNotNull("anonymous ID", anonymousBoltId);
	Bolt anonymousBoltSpec = topologyBolts.get(anonymousBoltId);
	assertNotNull("anonymous spec", anonymousBoltSpec);

	assertEquals("s1 parralelism", 1, spoutSpec.get_common().get_parallelism_hint());
	assertEquals("b1 parralelism", 1, boltSpec.get_common().get_parallelism_hint());
	assertEquals("second bold parrallelism", 2, anonymousBoltSpec.get_common().get_parallelism_hint());

	Map<GlobalStreamId,Grouping> boltInputs = boltSpec.get_common().get_inputs();
	assertEquals("input size", 1, boltInputs.size());
	GlobalStreamId streamId = boltInputs.keySet().iterator().next();
	assertEquals("input component id", "s1", streamId.get_componentId());
	assertEquals("input stream id", "default", streamId.get_streamId());
}