org.apache.storm.topology.BoltDeclarer Java Examples

private StormTopology build() {
	run();
	verify();

	Map<String,BoltDeclarer> declaredBolts = new HashMap<>();

	TopologyBuilder builder = new TopologyBuilder();
	for (Map.Entry<ConfiguredSpout,List<ConfiguredBolt>> line : entrySet()) {
		ConfiguredSpout spout = line.getKey();
		String lastId = spout.getId();
		String streamId = spout.getOutputStreamId();
		builder.setSpout(lastId, spout, spout.getParallelism());
		for (ConfiguredBolt bolt : line.getValue()) {
			String id = bolt.getId();
			BoltDeclarer declarer = declaredBolts.get(id);
			if (declarer == null)
				declarer = builder.setBolt(id, bolt, bolt.getParallelism());
			declarer.noneGrouping(lastId, streamId);
			if (declaredBolts.put(id, declarer) != null) break;
			lastId = id;
			streamId = bolt.getOutputStreamId();
		}
	}

	return builder.createTopology();
}
 

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

/***
 * Creates Pirk topology: KafkaSpout -> PartitionDataBolt -> EncRowCalcBolt -> EncColMultBolt -> OutputBolt Requires KafkaConfig to initialize KafkaSpout.
 *
 * @param kafkaConfig
 * @return
 */
public static StormTopology getPirkTopology(SpoutConfig kafkaConfig)
{
  // Create spout and bolts
  KafkaSpout spout = new KafkaSpout(kafkaConfig);
  PartitionDataBolt partitionDataBolt = new PartitionDataBolt();
  EncRowCalcBolt ercbolt = new EncRowCalcBolt();
  EncColMultBolt ecmbolt = new EncColMultBolt();
  OutputBolt outputBolt = new OutputBolt();

  // Build Storm topology
  TopologyBuilder builder = new TopologyBuilder();
  builder.setSpout(StormConstants.SPOUT_ID, spout, spoutParallelism);

  builder.setBolt(StormConstants.PARTITION_DATA_BOLT_ID, partitionDataBolt, partitionDataBoltParallelism).fieldsGrouping(StormConstants.SPOUT_ID,
      new Fields(StormConstants.HASH_FIELD));

  // TODO: Decide whether to use Resource Aware Scheduler. (If not, get rid of b2 and b3).
  BoltDeclarer b2 = builder.setBolt(StormConstants.ENCROWCALCBOLT_ID, ercbolt, encrowcalcboltParallelism)
      .fieldsGrouping(StormConstants.PARTITION_DATA_BOLT_ID, new Fields(StormConstants.HASH_FIELD))
      .allGrouping(StormConstants.ENCCOLMULTBOLT_ID, StormConstants.ENCCOLMULTBOLT_SESSION_END)
      .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, Integer.parseInt(SystemConfiguration.getProperty("storm.encrowcalcbolt.ticktuple")));

  // b2.setMemoryLoad(5000);
  // b2.setCPULoad(150.0);

  BoltDeclarer b3 = builder.setBolt(StormConstants.ENCCOLMULTBOLT_ID, ecmbolt, enccolmultboltParallelism)
      .fieldsGrouping(StormConstants.ENCROWCALCBOLT_ID, StormConstants.ENCROWCALCBOLT_DATASTREAM_ID,
          new Fields(StormConstants.COLUMN_INDEX_ERC_FIELD, StormConstants.SALT))
      .allGrouping(StormConstants.ENCROWCALCBOLT_ID, StormConstants.ENCROWCALCBOLT_FLUSH_SIG);
  // b3.setMemoryLoad(5000);
  // b3.setCPULoad(500.0);

  builder.setBolt(StormConstants.OUTPUTBOLT_ID, outputBolt, 1).globalGrouping(StormConstants.ENCCOLMULTBOLT_ID, StormConstants.ENCCOLMULTBOLT_ID);

  return builder.createTopology();
}
 

@Override
public BoltDeclarer fieldsGrouping(String componentId, String streamId, Fields fields) {
  this.addGrouping(
      componentId,
      streamId,
      GroupingTechnique.FIELD
  ).setGroupingKey(fields);
  this.madeASourceListener.onMadeASource(componentId);
  return this;
}
 

@Override
public BoltDeclarer customGrouping(String componentId,
                                   String streamId,
                                   CustomStreamGrouping grouping) {
  //todo
  return this;
}
 

@Override
public BoltDeclarer fieldsGrouping(String componentId, String streamId, Fields fields) {
    Pair<String, String> key = ImmutablePair.of(componentId, streamId);
    List<Fields> existing = fieldsGroupings.get(key);
    if (existing == null) {
        existing = new ArrayList<>();
    }
    existing.add(fields);
    fieldsGroupings.put(key, existing);
    return this;
}
 

@Override
public BoltDeclarer noneGrouping(String componentId, String streamId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer setMaxTaskParallelism(Number val) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer noneGrouping(String componentId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer localOrShuffleGrouping(String componentId, String streamId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer localOrShuffleGrouping(String componentId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer globalGrouping(String componentId, String streamId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer globalGrouping(String componentId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer fieldsGrouping(String componentId, Fields fields) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer setNumTasks(Number val) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer setMaxSpoutPending(Number val) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer addConfigurations(Map<String, Object> conf) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer allGrouping(String componentId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer directGrouping(String componentId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer directGrouping(String componentId, String streamId) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer partialKeyGrouping(String componentId, Fields fields) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer partialKeyGrouping(String componentId, String streamId, Fields fields) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer customGrouping(String componentId, CustomStreamGrouping grouping) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer customGrouping(String componentId, String streamId, CustomStreamGrouping grouping) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer setMemoryLoad(Number onHeap) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer addSharedMemory(SharedMemory request) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer addResources(Map<String, Double> resources) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer addResource(String resourceName, Number resourceValue) {
    throw new UnsupportedOperationException();
}
 

@Override
public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism) throws IllegalArgumentException {
    CustomBoltDeclarer declarer = new CustomBoltDeclarer(id, bolt, parallelism);
    createdBolts.add(declarer);
    return declarer;
}
 

protected void buildStreams(EcoExecutionContext executionContext, TopologyBuilder builder,
                            ObjectBuilder objectBuilder)
    throws IllegalAccessException, InstantiationException, ClassNotFoundException,
    NoSuchFieldException, InvocationTargetException {
  EcoTopologyDefinition topologyDefinition = executionContext.getTopologyDefinition();
  Map<String, ComponentStream> componentStreams = new HashMap<>();

  HashMap<String, BoltDeclarer> declarers = new HashMap<>();
  for (StreamDefinition stream : topologyDefinition.getStreams()) {
    Object boltObj = executionContext.getBolt(stream.getTo());
    BoltDeclarer declarer = declarers.get(stream.getTo());
    if (boltObj instanceof IRichBolt) {
      if (declarer == null) {
        declarer = builder.setBolt(stream.getTo(),
            (IRichBolt) boltObj,
            topologyDefinition.parallelismForBolt(stream.getTo()));
        declarers.put(stream.getTo(), declarer);
      }
    } else if (boltObj instanceof IBasicBolt) {
      if (declarer == null) {
        declarer = builder.setBolt(
            stream.getTo(),
            (IBasicBolt) boltObj,
            topologyDefinition.parallelismForBolt(stream.getTo()));
        declarers.put(stream.getTo(), declarer);
      }
    } else if (boltObj instanceof IWindowedBolt) {
      if (declarer == null) {
        declarer = builder.setBolt(
            stream.getTo(),
            (IWindowedBolt) boltObj,
            topologyDefinition.parallelismForBolt(stream.getTo()));
        declarers.put(stream.getTo(), declarer);
      }
    }  else {
      throw new IllegalArgumentException("Class does not appear to be a bolt: "
          + boltObj.getClass().getName());
    }

    GroupingDefinition grouping = stream.getGrouping();
    // if the streamId is defined, use it for the grouping,
    // otherwise assume default stream
    String streamId = grouping.getStreamId() == null
        ? Utils.DEFAULT_STREAM_ID : grouping.getStreamId();


    switch (grouping.getType()) {
      case SHUFFLE:
        declarer.shuffleGrouping(stream.getFrom(), streamId);
        break;
      case FIELDS:
        List<String> groupingArgs = grouping.getArgs();
        if (groupingArgs == null) {
          throw new IllegalArgumentException("You must supply arguments for Fields grouping");
        }
        declarer.fieldsGrouping(stream.getFrom(), streamId, new Fields(groupingArgs));
        break;
      case ALL:
        declarer.allGrouping(stream.getFrom(), streamId);
        break;
      case GLOBAL:
        declarer.globalGrouping(stream.getFrom(), streamId);
        break;
      case NONE:
        declarer.noneGrouping(stream.getFrom(), streamId);
        break;
      case CUSTOM:
        declarer.customGrouping(stream.getFrom(), streamId,
            buildCustomStreamGrouping(stream.getGrouping().getCustomClass(),
                executionContext,
                objectBuilder));
        break;
      default:
        throw new UnsupportedOperationException("unsupported grouping type: " + grouping);
    }
  }
  executionContext.setStreams(componentStreams);
}