backtype.storm.topology.BoltDeclarer Java Examples

The following examples show how to use backtype.storm.topology.BoltDeclarer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #2

Source File: StormProcessingItemTest.java From samoa with Apache License 2.0

6 votes

@Test
public void testAddToTopology() {
    new Expectations() {
        {
            topology.getStormBuilder();
            result = stormBuilder;

            stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
            result = new MockUp<BoltDeclarer>() {
            }.getMockInstance();
        }
    };

    pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

    new Verifications() {
        {
            assertEquals(pi.getProcessor(), processor);
            // TODO add methods to explore a topology and verify them
            assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
            assertEquals(pi.getId(), ID);
        }
    };
}

Example #3

Source File: TransactionTopologyBuilder.java From jstorm with Apache License 2.0

6 votes

@Override
public BoltDeclarer setBolt(String id, IRichBolt bolt, Number parallelismHint) throws IllegalArgumentException{
    upToDownstreamComponentsMap.put(id, new HashSet<String>());
    validateUnusedId(id);
    IRichBolt boltExecutor;
    boolean isStatefulBolt = false;
    if (bolt instanceof ITransactionStatefulBoltExecutor) {
        isStatefulBolt = true;
        boltExecutor = new TransactionStatefulBolt((ITransactionStatefulBoltExecutor) bolt);
    } else {
        boltExecutor = new TransactionBolt((ITransactionBoltExecutor) bolt);
    }
    initCommon(id, boltExecutor, parallelismHint);
    _bolts.put(id, boltExecutor);
    BoltDeclarer ret = new TransactionBoltDeclarer(id);
    ret.addConfiguration(TransactionCommon.TRANSACTION_STATEFUL_BOLT, isStatefulBolt);

    // If using KvState bolt, the corresponding init operater would be registered here.
    if (bolt instanceof KvStatefulBoltExecutor) {
        ConfigExtension.registerTransactionTaskStateInitOp(TopologyBuilder.getStormConf(), id, KeyRangeStateTaskInit.class);
    }

    return ret;
}

Example #4

Source File: StormProcessingItemTest.java From incubator-samoa with Apache License 2.0

6 votes

@Test
public void testAddToTopology() {
  new Expectations() {
    {
      topology.getStormBuilder();
      result = stormBuilder;

      stormBuilder.setBolt(ID, (IRichBolt) any, anyInt);
      result = new MockUp<BoltDeclarer>() {
      }.getMockInstance();
    }
  };

  pi.addToTopology(topology, PARRALLELISM_HINT_4); // this parallelism hint is ignored

  new Verifications() {
    {
      assertEquals(pi.getProcessor(), processor);
      // TODO add methods to explore a topology and verify them
      assertEquals(pi.getParallelism(), PARRALLELISM_HINT_2);
      assertEquals(pi.getId(), ID);
    }
  };
}

Example #5

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws AlreadyAliveException, InvalidTopologyException, TopologyAssignException {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = JStormUtils.parseInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    conf.put(Config.STORM_CLUSTER_MODE, "distributed");
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #6

Source File: BatchTopologyBuilder.java From jstorm with Apache License 2.0

6 votes

public BoltDeclarer setBolt(String id, IBasicBolt bolt, int parallel) {
    CoordinatedBolt coordinatedBolt = new CoordinatedBolt(bolt);

    BoltDeclarer boltDeclarer = topologyBuilder.setBolt(id, coordinatedBolt, parallel);

    if (bolt instanceof IPrepareCommit) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.PREPARE_STREAM_ID);
    }

    if (bolt instanceof ICommitter) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.COMMIT_STREAM_ID);
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.REVERT_STREAM_ID);
    }

    if (bolt instanceof IPostCommit) {
        boltDeclarer.allGrouping(BatchDef.SPOUT_TRIGGER, BatchDef.POST_STREAM_ID);
    }

    return boltDeclarer;
}

Example #7

Source File: GCLogApplication.java From eagle with Apache License 2.0

6 votes

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfAnalyzerTasks = config.getInt(ANALYZER_TASK_NUM);
    int numOfGeneratorTasks = config.getInt(GENERATOR_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);

    GCLogAnalyzerBolt bolt = new GCLogAnalyzerBolt();
    BoltDeclarer boltDeclarer = builder.setBolt("analyzerBolt", bolt, numOfAnalyzerTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    GCMetricGeneratorBolt generatorBolt = new GCMetricGeneratorBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("generatorBolt", generatorBolt, numOfGeneratorTasks);
    joinBoltDeclarer.fieldsGrouping("analyzerBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("gc_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("generatorBolt", new Fields("f1"));
    return builder.createTopology();
}

Example #8

Source File: PerformanceTestTopology.java From jstorm with Apache License 2.0

6 votes

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}

Example #9

Source File: HdfsAuthLogMonitoringMain.java From eagle with Apache License 2.0

6 votes

public static void main(String[] args) throws Exception{
    System.setProperty("config.resource", "/application.conf");
    Config config = ConfigFactory.load();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    SecurityLogParserBolt bolt = new SecurityLogParserBolt();
    TopologyBuilder builder = new TopologyBuilder();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.shuffleGrouping("ingest");

    KafkaBolt kafkaBolt = new KafkaBolt();
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", kafkaBolt, numOfSinkTasks);
    kafkaBoltDeclarer.shuffleGrouping("parserBolt");

    StormTopology topology = builder.createTopology();

    TopologySubmitter.submit(topology, config);
}

Example #10

Source File: EagleMetricCollectorApplication.java From eagle with Apache License 2.0

6 votes

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    String deserClsName = config.getString("dataSourceConfig.deserializerClass");
    KafkaSourcedSpoutScheme scheme = new KafkaSourcedSpoutScheme(deserClsName, config);

    TopologyBuilder builder = new TopologyBuilder();
    BaseRichSpout spout1 = new KafkaOffsetSourceSpoutProvider().getSpout(config);
    BaseRichSpout spout2 = KafkaSourcedSpoutProvider.getSpout(config, scheme);

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfDistributionTasks = config.getInt(DISTRIBUTION_TASK_NUM);

    builder.setSpout("kafkaLogLagChecker", spout1, numOfSpoutTasks);
    builder.setSpout("kafkaMessageFetcher", spout2, numOfSpoutTasks);

    KafkaMessageDistributionBolt bolt = new KafkaMessageDistributionBolt(config);
    BoltDeclarer bolteclarer = builder.setBolt("distributionBolt", bolt, numOfDistributionTasks);
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    bolteclarer.fieldsGrouping("kafkaLogLagChecker", new Fields("f1"));
    return builder.createTopology();
}

Example #11

Source File: HBaseAuditLogApplication.java From eagle with Apache License 2.0

6 votes

@Override
public StormTopology execute(Config config, StormEnvironment environment) {
    TopologyBuilder builder = new TopologyBuilder();
    KafkaSpoutProvider provider = new KafkaSpoutProvider();
    IRichSpout spout = provider.getSpout(config);

    HBaseAuditLogParserBolt bolt = new HBaseAuditLogParserBolt();

    int numOfSpoutTasks = config.getInt(SPOUT_TASK_NUM);
    int numOfParserTasks = config.getInt(PARSER_TASK_NUM);
    int numOfJoinTasks = config.getInt(JOIN_TASK_NUM);
    int numOfSinkTasks = config.getInt(SINK_TASK_NUM);

    builder.setSpout("ingest", spout, numOfSpoutTasks);
    BoltDeclarer boltDeclarer = builder.setBolt("parserBolt", bolt, numOfParserTasks);
    boltDeclarer.fieldsGrouping("ingest", new Fields(StringScheme.STRING_SCHEME_KEY));

    HBaseResourceSensitivityDataJoinBolt joinBolt = new HBaseResourceSensitivityDataJoinBolt(config);
    BoltDeclarer joinBoltDeclarer = builder.setBolt("joinBolt", joinBolt, numOfJoinTasks);
    joinBoltDeclarer.fieldsGrouping("parserBolt", new Fields("f1"));

    StormStreamSink sinkBolt = environment.getStreamSink("hbase_audit_log_stream",config);
    BoltDeclarer kafkaBoltDeclarer = builder.setBolt("kafkaSink", sinkBolt, numOfSinkTasks);
    kafkaBoltDeclarer.fieldsGrouping("joinBolt", new Fields("user"));
    return builder.createTopology();
}

Example #12

Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer noneGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #13

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer directGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.directGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #14

Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localFirstGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #15

Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer shuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #16

Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localOrShuffleGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #17

Source File: TransactionalTopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localOrShuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #18

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer fieldsGrouping(final String component, final String streamId, final Fields fields) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.fieldsGrouping(component, streamId, fields);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #19

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer allGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.allGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #20

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer allGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.allGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #21

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer noneGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #22

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer noneGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.noneGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #23

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localFirstGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #24

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localFirstGrouping(final String componentId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localFirstGrouping(componentId);
        }

        @Override
        public String getComponent() {
            return componentId;
        }
    });
    return this;
}

Example #25

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer localOrShuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.localOrShuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #26

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer shuffleGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #27

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer shuffleGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.shuffleGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #28

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer globalGrouping(final String component, final String streamId) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.globalGrouping(component, streamId);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #29

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer globalGrouping(final String component) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.globalGrouping(component);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}

Example #30

Source File: BatchSubtopologyBuilder.java From jstorm with Apache License 2.0

5 votes

@Override
public BoltDeclarer fieldsGrouping(final String component, final Fields fields) {
    addDeclaration(new InputDeclaration() {
        @Override
        public void declare(InputDeclarer declarer) {
            declarer.fieldsGrouping(component, fields);
        }

        @Override
        public String getComponent() {
            return component;
        }
    });
    return this;
}