org.apache.storm.generated.Bolt Java Examples

public static Set<String> getTerminalUserBoltNames(StormTopology topology) {
    Set<String> terminalBolts = new HashSet<>();
    Set<String> inputs = new HashSet<>();
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String name = entry.getKey();
        Set<GlobalStreamId> inputsForBolt = entry.getValue().get_common().get_inputs().keySet();
        if (!isSystemComponent(name)) {
            for (GlobalStreamId streamId : inputsForBolt) {
                inputs.add(streamId.get_componentId());
            }
        }
    }

    for (String boltName : topology.get_bolts().keySet()) {
        if (!isSystemComponent(boltName) && !inputs.contains(boltName)) {
            terminalBolts.add(boltName);
        }
    }

    return terminalBolts;
}
 

public static Map<String, Set<String>> getAdjacencyMap(StormTopology topology,
                                                       boolean removeSystemComponent) {
    Map<String, Set<String>> adjacencyMap = new HashMap<>();

    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltName = entry.getKey();
        Map<GlobalStreamId, Grouping> inputs = entry.getValue().get_common().get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String inputComponentId = input.getKey().get_componentId();
            Set<String> components = adjacencyMap.containsKey(inputComponentId)
                    ? adjacencyMap.get(inputComponentId) : new HashSet<String>();
            components.add(boltName);
            components = removeSystemComponent ? removeSystemComponents(components)
                    : components;
            if (!removeSystemComponent || !isSystemComponent(inputComponentId)) {
                adjacencyMap.put(inputComponentId, components);
            }
        }
    }

    return adjacencyMap;
}
 

private void addTopologyOutputs(StormTopology stormTopology, String topologyOwner, Map stormConf, AtlasEntity topology, AtlasEntityExtInfo entityExtInfo) {
    List<AtlasEntity> outputs   = new ArrayList<>();
    Map<String, Bolt> bolts     = stormTopology.get_bolts();
    Set<String>       boltNames = StormTopologyUtil.getTerminalUserBoltNames(stormTopology);

    for (String boltName : boltNames) {
        Serializable instance = Utils.javaDeserialize(bolts.get(boltName).get_bolt_object().get_serialized_java(), Serializable.class);
        String       dsType   = instance.getClass().getSimpleName();
        AtlasEntity  dsEntity = addDataSet(dsType, topologyOwner, instance, stormConf, entityExtInfo);

        if (dsEntity != null) {
            outputs.add(dsEntity);
        }
    }

    topology.setRelationshipAttribute("outputs", AtlasTypeUtil.getAtlasRelatedObjectIds(outputs, RELATIONSHIP_PROCESS_DATASET_OUTPUTS));
}
 

/**
 * Gets the Thrift object representing the topology.
 *
 * @return the Thrift definition representing the topology
 */
@SuppressWarnings("deprecation")
public StormTopology getRawTopology() {

  StormTopology stormTopology = new StormTopology();
  Map<String, SpoutSpec> spouts = new HashMap<>();
  for (TopologyAPI.Spout spout : this.delegate.getRawTopology().getSpoutsList()) {
    spouts.put(spout.getComp().getName(), new SpoutSpec(spout));
  }

  Map<String, Bolt> bolts = new HashMap<>();
  for (TopologyAPI.Bolt bolt : this.delegate.getRawTopology().getBoltsList()) {
    bolts.put(bolt.getComp().getName(), new Bolt(bolt));
  }

  stormTopology.set_spouts(spouts);
  stormTopology.set_bolts(bolts);
  return stormTopology;
}
 

public static Set<String> getTerminalUserBoltNames(StormTopology topology) {
    Set<String> terminalBolts = new HashSet<>();
    Set<String> inputs = new HashSet<>();
    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String name = entry.getKey();
        Set<GlobalStreamId> inputsForBolt = entry.getValue().get_common().get_inputs().keySet();
        if (!isSystemComponent(name)) {
            for (GlobalStreamId streamId : inputsForBolt) {
                inputs.add(streamId.get_componentId());
            }
        }
    }

    for (String boltName : topology.get_bolts().keySet()) {
        if (!isSystemComponent(boltName) && !inputs.contains(boltName)) {
            terminalBolts.add(boltName);
        }
    }

    return terminalBolts;
}
 

public static Map<String, Set<String>> getAdjacencyMap(StormTopology topology,
                                                       boolean removeSystemComponent)
throws Exception {
    Map<String, Set<String>> adjacencyMap = new HashMap<>();

    for (Map.Entry<String, Bolt> entry : topology.get_bolts().entrySet()) {
        String boltName = entry.getKey();
        Map<GlobalStreamId, Grouping> inputs = entry.getValue().get_common().get_inputs();
        for (Map.Entry<GlobalStreamId, Grouping> input : inputs.entrySet()) {
            String inputComponentId = input.getKey().get_componentId();
            Set<String> components = adjacencyMap.containsKey(inputComponentId)
                    ? adjacencyMap.get(inputComponentId) : new HashSet<String>();
            components.add(boltName);
            components = removeSystemComponent ? removeSystemComponents(components)
                    : components;
            if (!removeSystemComponent || !isSystemComponent(inputComponentId)) {
                adjacencyMap.put(inputComponentId, components);
            }
        }
    }

    return adjacencyMap;
}
 

private void addTopologyOutputs(Referenceable topologyReferenceable,
                                StormTopology stormTopology, String topologyOwner,
                                Map stormConf, List<Referenceable> dependentEntities) throws Exception {
    final ArrayList<Referenceable> outputDataSets = new ArrayList<>();

    Map<String, Bolt> bolts = stormTopology.get_bolts();
    Set<String> terminalBoltNames = StormTopologyUtil.getTerminalUserBoltNames(stormTopology);
    for (String terminalBoltName : terminalBoltNames) {
        Serializable instance = Utils.javaDeserialize(bolts.get(terminalBoltName)
                .get_bolt_object().get_serialized_java(), Serializable.class);

        String dataSetType = instance.getClass().getSimpleName();
        final Referenceable datasetRef = createDataSet(dataSetType, topologyOwner, instance, stormConf, dependentEntities);
        if (datasetRef != null) {
            outputDataSets.add(datasetRef);
        }
    }

    topologyReferenceable.set("outputs", outputDataSets);
}
 

@Test
public void aggregate() throws Exception {
	beansXml = "<breeze:topology id='aggregate'>" +
			"<breeze:spout id='iterator' beanType='java.util.Iterator' signature='next()' outputFields='x'/>" +
			"<breeze:spout id='queue' beanType='java.util.Queue' signature='poll()' outputFields='x'/>" +
			"<breeze:bolt id='collector' beanType='org.slf4j.Logger' signature='info(x)'/>" +
			"</breeze:topology>";
	refresh();

	StormTopology topology = getBean("aggregate", StormTopology.class);

	Bolt collector = topology.get_bolts().get("collector");
	Map<GlobalStreamId, Grouping> inputs = collector.get_common().get_inputs();
	assertEquals("input count", 2, inputs.size());
	assertNotNull("iterator grouping", inputs.get(new GlobalStreamId("iterator", "default")));
	assertNotNull("queue grouping", inputs.get(new GlobalStreamId("queue", "default")));
}
 

private List<AtlasEntity> createTopologyGraph(StormTopology stormTopology, Map<String, SpoutSpec> spouts, Map<String, Bolt> bolts) {
    // Add graph of nodes in the topology
    Map<String, AtlasEntity> nodeEntities = new HashMap<>();

    addSpouts(spouts, nodeEntities);
    addBolts(bolts, nodeEntities);

    addGraphConnections(stormTopology, nodeEntities);

    return new ArrayList<>(nodeEntities.values());
}
 

private void addBolts(Map<String, Bolt> bolts, Map<String, AtlasEntity> nodeEntities) {
    for (Map.Entry<String, Bolt> entry : bolts.entrySet()) {
        String      boltName     = entry.getKey();
        AtlasEntity boltInstance = createBoltInstance(boltName, entry.getValue());

        nodeEntities.put(boltName, boltInstance);
    }
}
 

private AtlasEntity createBoltInstance(String boltName, Bolt stormBolt) {
    AtlasEntity         bolt          = new AtlasEntity(StormDataTypes.STORM_BOLT.getName());
    Serializable        instance      = Utils.javaDeserialize(stormBolt.get_bolt_object().get_serialized_java(), Serializable.class);
    Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);

    bolt.setAttribute(AtlasClient.NAME, boltName);
    bolt.setAttribute("driverClass", instance.getClass().getName());
    bolt.setAttribute("conf", flatConfigMap);

    return bolt;
}
 

private ArrayList<Referenceable> createTopologyGraph(StormTopology stormTopology,
                                                     Map<String, SpoutSpec> spouts,
                                                     Map<String, Bolt> bolts) throws Exception {
    // Add graph of nodes in the topology
    final Map<String, Referenceable> nodeEntities = new HashMap<>();
    addSpouts(spouts, nodeEntities);
    addBolts(bolts, nodeEntities);

    addGraphConnections(stormTopology, nodeEntities);

    ArrayList<Referenceable> nodes = new ArrayList<>();
    nodes.addAll(nodeEntities.values());
    return nodes;
}
 

private void addBolts(Map<String, Bolt> bolts,
                      Map<String, Referenceable> nodeEntities) throws IllegalAccessException {
    for (Map.Entry<String, Bolt> entry : bolts.entrySet()) {
        Referenceable boltInstance = createBoltInstance(entry.getKey(), entry.getValue());
        nodeEntities.put(entry.getKey(), boltInstance);
    }
}
 

private Referenceable createBoltInstance(String boltName,
                                         Bolt stormBolt) throws IllegalAccessException {
    Referenceable boltReferenceable = new Referenceable(StormDataTypes.STORM_BOLT.getName());

    boltReferenceable.set(AtlasClient.NAME, boltName);

    Serializable instance = Utils.javaDeserialize(
            stormBolt.get_bolt_object().get_serialized_java(), Serializable.class);
    boltReferenceable.set("driverClass", instance.getClass().getName());

    Map<String, String> flatConfigMap = StormTopologyUtil.getFieldValues(instance, true, null);
    boltReferenceable.set("conf", flatConfigMap);

    return boltReferenceable;
}
 

@Test
public void build() throws Exception {
	beansXml = "<breeze:topology id='t1'>" +
			"<breeze:spout id='s1' beanType='eu.icolumbo.breeze.TestBean' signature='ping()' outputFields='feed'/>" +
			"<breeze:bolt id='b1' beanType='eu.icolumbo.breeze.TestBean' signature='echo(feed)' outputFields='replay' scatterOutput='true'/>" +
			"<breeze:bolt beanType='eu.icolumbo.breeze.TestBean' signature='drain(replay)' parallelism='2'/>" +
			"</breeze:topology>";
	refresh();

	StormTopology topology = getBean("t1", StormTopology.class);
	assertEquals("spout count", 1, topology.get_spouts_size());
	assertEquals("bolt count", 2, topology.get_bolts_size());

	SpringSpout spout = getBean("s1", SpringSpout.class);
	assertEquals("spout ID", "s1", spout.getId());
	assertEquals("spout scatter", false, spout.getScatterOutput());
	SpringBolt bolt = getBean("b1", SpringBolt.class);
	assertEquals("bolt ID", "b1", bolt.getId());
	assertEquals("bolt scatter", true, bolt.getScatterOutput());

	Map<String, SpoutSpec> topologySpouts = topology.get_spouts();
	SpoutSpec spoutSpec = topologySpouts.get("s1");
	assertNotNull("s1 spec", spoutSpec);

	Map<String, Bolt> topologyBolts = topology.get_bolts();
	Bolt boltSpec = topologyBolts.get("b1");
	assertNotNull("b1 spec", boltSpec);

	String anonymousBoltId = null;
	for (String id : topologyBolts.keySet())
		if (! "b1".equals(id))
			anonymousBoltId = id;
	assertNotNull("anonymous ID", anonymousBoltId);
	Bolt anonymousBoltSpec = topologyBolts.get(anonymousBoltId);
	assertNotNull("anonymous spec", anonymousBoltSpec);

	assertEquals("s1 parralelism", 1, spoutSpec.get_common().get_parallelism_hint());
	assertEquals("b1 parralelism", 1, boltSpec.get_common().get_parallelism_hint());
	assertEquals("second bold parrallelism", 2, anonymousBoltSpec.get_common().get_parallelism_hint());

	Map<GlobalStreamId,Grouping> boltInputs = boltSpec.get_common().get_inputs();
	assertEquals("input size", 1, boltInputs.size());
	GlobalStreamId streamId = boltInputs.keySet().iterator().next();
	assertEquals("input component id", "s1", streamId.get_componentId());
	assertEquals("input stream id", "default", streamId.get_streamId());
}