backtype.storm.topology.IRichBolt Java Examples
The following examples show how to use
backtype.storm.topology.IRichBolt.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StormProcessingItemTest.java From incubator-samoa with Apache License 2.0 | 6 votes |
@Test public void testAddToTopology() { new Expectations() { { topology.getStormBuilder(); result = stormBuilder; stormBuilder.setBolt(ID, (IRichBolt) any, anyInt); result = new MockUp<BoltDeclarer>() { }.getMockInstance(); } }; pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored new Verifications() { { assertEquals(pi.getProcessor(), processor); // TODO add methods to explore a topology and verify them assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2); assertEquals(pi.getId(), ID); } }; }
Example #2
Source File: StormProcessingItemTest.java From samoa with Apache License 2.0 | 6 votes |
@Test public void testAddToTopology() { new Expectations() { { topology.getStormBuilder(); result = stormBuilder; stormBuilder.setBolt(ID, (IRichBolt) any, anyInt); result = new MockUp<BoltDeclarer>() { }.getMockInstance(); } }; pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored new Verifications() { { assertEquals(pi.getProcessor(), processor); // TODO add methods to explore a topology and verify them assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2); assertEquals(pi.getId(), ID); } }; }
Example #3
Source File: TransactionTopologyBuilder.java From jstorm with Apache License 2.0 | 6 votes |
@Override public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException{ upToDownstreamComponentsMap.put(id, new HashSet<String>()); validateUnusedId(id); IRichBolt boltExecutor; boolean isStatefulBolt = false; if (bolt instanceof ITransactionStatefulBoltExecutor) { isStatefulBolt = true; boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt); } else { boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt); } initCommon(id, boltExecutor, parallelismHint); _bolts.put(id, boltExecutor); BoltDeclarer ret = new TransactionBoltDeclarer(id); ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt); // If using KvState bolt, the corresponding init operater would be registered here. if (bolt instanceof KvStatefulBoltExecutor) { ConfigExtension.registerTransactionTaskStateInitOp(TopologyBuilder.getStormConf(), id, KeyRangeStateTaskInit.class); } return ret; }
Example #4
Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0 | 5 votes |
private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism) { Integer p = null; if (parallelism != null) p = parallelism.intValue(); Component component = new Component(bolt, p); _bolts.put(id, component); return new BoltDeclarerImpl(component); }
Example #5
Source File: ComponentCreator.java From PoseidonX with Apache License 2.0 | 5 votes |
/** * 根据Streaming算子创建Storm Bolt实例 * */ public static IRichBolt createBolt(IRichOperator operator, StormConf stormConf) throws StreamingException { if (operator instanceof OutputOperator) { return createOutputBolt((OutputOperator)operator, stormConf); } return CommonBoltCreator.createFunctionBolt(operator); }
Example #6
Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0 | 5 votes |
private BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelism, boolean committer) { Integer p = null; if (parallelism != null) p = parallelism.intValue(); Component component = new Component(bolt, p, committer); _bolts.put(id, component); return new BoltDeclarerImpl(component); }
Example #7
Source File: CoordinatedBolt.java From jstorm with Apache License 2.0 | 5 votes |
public CoordinatedBolt(IRichBolt delegate, Map<String, SourceArgs> sourceArgs, IdStreamSpec idStreamSpec) { _sourceArgs = sourceArgs; if (_sourceArgs == null) _sourceArgs = new HashMap<>(); _delegate = delegate; _idStreamSpec = idStreamSpec; }
Example #8
Source File: CommonBoltCreator.java From PoseidonX with Apache License 2.0 | 5 votes |
/** * 创建输出算子 */ public static IRichBolt createOutputBolt(OutputOperator operator) { StormOutputBolt outputBolt = new StormOutputBolt(); outputBolt.setOperator(operator); return outputBolt; }
Example #9
Source File: FlowmixBuilder.java From flowmix with Apache License 2.0 | 5 votes |
/** * @return A topology builder than can further be customized. */ public TopologyBuilder create() { TopologyBuilder builder = new TopologyBuilder(); if(eventsComponent instanceof IRichSpout) builder.setSpout(EVENT, (IRichSpout) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism); else if(eventsComponent instanceof IRichBolt) builder.setBolt(EVENT, (IRichBolt) eventsComponent, eventLoaderParallelism == -1 ? parallelismHint : eventLoaderParallelism); else throw new RuntimeException("The component for events is not valid. Must be IRichSpout or IRichBolt"); if(flowLoaderSpout instanceof IRichSpout) builder.setSpout(FLOW_LOADER_STREAM, (IRichSpout) flowLoaderSpout, 1); else if(flowLoaderSpout instanceof IRichBolt) builder.setBolt(FLOW_LOADER_STREAM, (IRichBolt) flowLoaderSpout, 1); else throw new RuntimeException("The component for rules is not valid. Must be IRichSpout or IRichBolt"); builder.setSpout("tick", new TickSpout(1000), 1); builder.setBolt(INITIALIZER, new FlowInitializerBolt(), parallelismHint) // kicks off a flow determining where to start .localOrShuffleGrouping(EVENT) .allGrouping(FLOW_LOADER_STREAM, FLOW_LOADER_STREAM); declarebolt(builder, FILTER, new FilterBolt(), parallelismHint, true); declarebolt(builder, SELECT, new SelectorBolt(), parallelismHint, true); declarebolt(builder, PARTITION, new PartitionBolt(), parallelismHint, true); declarebolt(builder, SWITCH, new SwitchBolt(), parallelismHint, true); declarebolt(builder, AGGREGATE, new AggregatorBolt(), parallelismHint, true); declarebolt(builder, JOIN, new JoinBolt(), parallelismHint, true); declarebolt(builder, EACH, new EachBolt(), parallelismHint, true); declarebolt(builder, SORT, new SortBolt(), parallelismHint, true); declarebolt(builder, SPLIT, new SplitBolt(), parallelismHint, true); declarebolt(builder, OUTPUT, outputBolt, parallelismHint, false); return builder; }
Example #10
Source File: FlowmixBuilder.java From flowmix with Apache License 2.0 | 5 votes |
private static void declarebolt(TopologyBuilder builder, String boltName, IRichBolt bolt, int parallelism, boolean control) { BoltDeclarer declarer = builder.setBolt(boltName, bolt, parallelism) .allGrouping(FLOW_LOADER_STREAM, FLOW_LOADER_STREAM) .allGrouping("tick", "tick") .localOrShuffleGrouping(INITIALIZER, boltName) .localOrShuffleGrouping(FILTER, boltName) .fieldsGrouping(PARTITION, boltName, new Fields(FLOW_ID, PARTITION)) // guaranteed partitions will always group the same flow for flows that have joins with default partitions. .localOrShuffleGrouping(AGGREGATE, boltName) .localOrShuffleGrouping(SELECT, boltName) .localOrShuffleGrouping(EACH, boltName) .localOrShuffleGrouping(SORT, boltName) .localOrShuffleGrouping(SWITCH, boltName) .localOrShuffleGrouping(SPLIT, boltName) .localOrShuffleGrouping(JOIN, boltName); if(control) { // control stream is all-grouped declarer.allGrouping(INITIALIZER, BROADCAST_STREAM + boltName) .allGrouping(FILTER, BROADCAST_STREAM + boltName) .allGrouping(PARTITION, BROADCAST_STREAM + boltName) .allGrouping(AGGREGATE, BROADCAST_STREAM + boltName) .allGrouping(SELECT, BROADCAST_STREAM + boltName) .allGrouping(EACH, BROADCAST_STREAM + boltName) .allGrouping(SORT, BROADCAST_STREAM + boltName) .allGrouping(SWITCH, BROADCAST_STREAM + boltName) .allGrouping(SPLIT, BROADCAST_STREAM + boltName) .allGrouping(JOIN, BROADCAST_STREAM + boltName); } }
Example #11
Source File: LinearDRPCTopologyBuilder.java From jstorm with Apache License 2.0 | 5 votes |
@Deprecated public LinearDRPCInputDeclarer addBolt(IRichBolt bolt, Number parallelism) { if (parallelism == null) parallelism = 1; Component component = new Component(bolt, parallelism.intValue()); _components.add(component); return new InputDeclarerImpl(component); }
Example #12
Source File: JStormApplication.java From PoseidonX with Apache License 2.0 | 5 votes |
private BoltDeclarer createBoltDeclarer(IRichOperator operator) { IRichBolt bolt; if ((operator instanceof FunctionOperator) || (operator instanceof FunctionStreamOperator)) { bolt = createStormBolt(operator); } else { bolt = createOutputStormBolt(operator); } return builder.setBolt(operator.getOperatorId(), bolt, operator.getParallelNumber()); }
Example #13
Source File: KeyedFairBolt.java From jstorm with Apache License 2.0 | 4 votes |
public KeyedFairBolt(IRichBolt delegate) { _delegate = delegate; }
Example #14
Source File: AckTransactionBolt.java From jstorm with Apache License 2.0 | 4 votes |
public AckTransactionBolt(IRichBolt bolt) { this.bolt = bolt; }
Example #15
Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
public Component(IRichBolt bolt, Integer parallelism) { this.bolt = bolt; this.parallelism = parallelism; }
Example #16
Source File: CoordinatedBolt.java From jstorm with Apache License 2.0 | 4 votes |
public CoordinatedBolt(IRichBolt delegate) { this(delegate, null, null); }
Example #17
Source File: CoordinatedBolt.java From jstorm with Apache License 2.0 | 4 votes |
public CoordinatedBolt(IRichBolt delegate, String sourceComponent, SourceArgs sourceArgs, IdStreamSpec idStreamSpec) { this(delegate, singleSourceArgs(sourceComponent, sourceArgs), idStreamSpec); }
Example #18
Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
public Component(IRichBolt bolt, Integer parallelism, boolean committer) { this.bolt = bolt; this.parallelism = parallelism; this.committer = committer; }
Example #19
Source File: BoltTracker.java From jstorm with Apache License 2.0 | 4 votes |
public BoltTracker(IRichBolt delegate, String id) { super(delegate, id); _richDelegate = delegate; }
Example #20
Source File: LinearDRPCTopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
public Component(IRichBolt bolt, int parallelism) { this.bolt = bolt; this.parallelism = parallelism; this.componentConfs = new ArrayList(); }
Example #21
Source File: LinearDRPCTopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
private StormTopology createTopology(DRPCSpout spout) { final String SPOUT_ID = "spout"; final String PREPARE_ID = "prepare-request"; TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SPOUT_ID, spout); builder.setBolt(PREPARE_ID, new PrepareRequest()).noneGrouping(SPOUT_ID); int i = 0; for (; i < _components.size(); i++) { Component component = _components.get(i); Map<String, SourceArgs> source = new HashMap<String, SourceArgs>(); if (i == 1) { source.put(boltId(i - 1), SourceArgs.single()); } else if (i >= 2) { source.put(boltId(i - 1), SourceArgs.all()); } IdStreamSpec idSpec = null; if (i == _components.size() - 1 && component.bolt instanceof FinishedCallback) { idSpec = IdStreamSpec.makeDetectSpec(PREPARE_ID, PrepareRequest.ID_STREAM); } BoltDeclarer declarer = builder.setBolt(boltId(i), new CoordinatedBolt(component.bolt, source, idSpec), component.parallelism); for (Map conf : component.componentConfs) { declarer.addConfigurations(conf); } if (idSpec != null) { declarer.fieldsGrouping(idSpec.getGlobalStreamId().get_componentId(), PrepareRequest.ID_STREAM, new Fields("request")); } if (i == 0 && component.declarations.isEmpty()) { declarer.noneGrouping(PREPARE_ID, PrepareRequest.ARGS_STREAM); } else { String prevId; if (i == 0) { prevId = PREPARE_ID; } else { prevId = boltId(i - 1); } for (InputDeclaration declaration : component.declarations) { declaration.declare(prevId, declarer); } } if (i > 0) { declarer.directGrouping(boltId(i - 1), Constants.COORDINATED_STREAM_ID); } } IRichBolt lastBolt = _components.get(_components.size() - 1).bolt; OutputFieldsGetter getter = new OutputFieldsGetter(); lastBolt.declareOutputFields(getter); Map<String, StreamInfo> streams = getter.getFieldsDeclaration(); if (streams.size() != 1) { throw new RuntimeException("Must declare exactly one stream from last bolt in LinearDRPCTopology"); } String outputStream = streams.keySet().iterator().next(); List<String> fields = streams.get(outputStream).get_output_fields(); if (fields.size() != 2) { throw new RuntimeException( "Output stream of last component in LinearDRPCTopology must contain exactly two fields. The first should be the request id, and the second should be the result."); } builder.setBolt("JoinResult", new JoinResult(PREPARE_ID)).fieldsGrouping(boltId(i - 1), outputStream, new Fields(fields.get(0))) .fieldsGrouping(PREPARE_ID, PrepareRequest.RETURN_STREAM, new Fields("request")); i++; builder.setBolt("ReturnResults", new ReturnResults()).noneGrouping("JoinResult"); return builder.createTopology(); }
Example #22
Source File: LinearDRPCTopologyBuilder.java From jstorm with Apache License 2.0 | 4 votes |
@Deprecated public LinearDRPCInputDeclarer addBolt(IRichBolt bolt) { return addBolt(bolt, null); }
Example #23
Source File: FlowmixBuilder.java From flowmix with Apache License 2.0 | 4 votes |
public FlowmixBuilder setOutputBolt(IRichBolt outputBolt) { this.outputBolt = outputBolt; return this; }
Example #24
Source File: CommonBoltCreator.java From PoseidonX with Apache License 2.0 | 4 votes |
/** * 创建功能性算子 */ public static IRichBolt createFunctionBolt(IRichOperator operator){ StormBolt bolt = new StormBolt(); bolt.setOperator(operator); return bolt; }
Example #25
Source File: ComponentCreator.java From PoseidonX with Apache License 2.0 | 4 votes |
private static IRichBolt createOutputBolt(OutputOperator operator, StormConf stormConf) throws StreamingException { return CommonBoltCreator.createOutputBolt(operator); }
Example #26
Source File: JStormApplication.java From PoseidonX with Apache License 2.0 | 4 votes |
private IRichBolt createStormBolt(IRichOperator f) { StormBolt stormbolt = new StormBolt(); stormbolt.setOperator(f); return stormbolt; }
Example #27
Source File: JStormApplication.java From PoseidonX with Apache License 2.0 | 4 votes |
private IRichBolt createOutputStormBolt(IRichOperator f) { StormOutputBolt outputbolt = new StormOutputBolt(); outputbolt.setOperator(f); return outputbolt; }
Example #28
Source File: TransactionTopologyBuilder.java From jstorm with Apache License 2.0 | 2 votes |
/** * Build bolt to provide the compatibility with Storm's ack mechanism * * @param id bolt Id * @param bolt * @return */ public BoltDeclarer setBoltWithAck(String id, IRichBolt bolt, Number parallelismHint) { return setBolt(id, new AckTransactionBolt(bolt), parallelismHint); }