org.apache.storm.tuple.Fields Java Examples

The following examples show how to use org.apache.storm.tuple.Fields. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: WordCountTopologyNode.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

        TopologyBuilder builder = new TopologyBuilder();

        builder.setSpout("spout", new RandomSentence(), 5);

        builder.setBolt("split", new SplitSentence(), 8).shuffleGrouping("spout");
        builder.setBolt("count", new WordCount(), 12).fieldsGrouping("split", new Fields("word"));

        Config conf = new Config();
        conf.setDebug(true);
        String topoName = "word-count";
        if (args != null && args.length > 0) {
            topoName = args[0];
        }
        conf.setNumWorkers(3);
        StormSubmitter.submitTopologyWithProgressBar(topoName, conf, builder.createTopology());
    }
 
Example #2
Source File: TridentHBaseWindowingStoreTopology.java    From storm-net-adapter with Apache License 2.0 6 votes vote down vote up
public static StormTopology buildTopology(WindowsStoreFactory windowsStore) throws Exception {
    FixedBatchSpout spout = new FixedBatchSpout(new Fields("sentence"), 3, new Values("the cow jumped over the moon"),
                                                new Values("the man went to the store and bought some candy"),
                                                new Values("four score and seven years ago"),
                                                new Values("how many apples can you eat"), new Values("to be or not to be the person"));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();

    Stream stream = topology.newStream("spout1", spout).parallelismHint(16).each(new Fields("sentence"),
                                                                                 new Split(), new Fields("word"))
                            .window(TumblingCountWindow.of(1000), windowsStore, new Fields("word"), new CountAsAggregator(),
                                    new Fields("count"))
                            .peek(new Consumer() {
                                @Override
                                public void accept(TridentTuple input) {
                                    LOG.info("Received tuple: [{}]", input);
                                }
                            });

    return topology.build();
}
 
Example #3
Source File: WorkerTopologyContext.java    From incubator-heron with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("rawtypes")
public WorkerTopologyContext(
    StormTopology topology,
    Map stormConf,
    Map<Integer, String> taskToComponent,
    Map<String, List<Integer>> componentToSortedTasks,
    Map<String, Map<String, Fields>> componentToStreamToFields,
    String stormId,
    String codeDir,
    String pidDir,
    Integer workerPort,
    List<Integer> workerTasks,
    Map<String, Object> defaultResources,
    Map<String, Object> userResources
) {
  super(topology, stormConf, taskToComponent,
      componentToSortedTasks, componentToStreamToFields, stormId);
  throw new RuntimeException("WorkerTopologyContext should never be init this way");
}
 
Example #4
Source File: TickSpoutTest.java    From bullet-storm with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeclaredOutputFields() {
    CustomOutputFieldsDeclarer declarer = new CustomOutputFieldsDeclarer();
    spout.declareOutputFields(declarer);
    Fields expected = new Fields(TopologyConstants.ID_FIELD, TopologyConstants.TICK_FIELD);
    Assert.assertTrue(declarer.areFieldsPresent(TopologyConstants.TICK_STREAM, false, expected));
}
 
Example #5
Source File: CrawlTopology.java    From storm-crawler with Apache License 2.0 5 votes vote down vote up
@Override
protected int run(String[] args) {
    TopologyBuilder builder = new TopologyBuilder();

    String[] testURLs = new String[] { "http://www.lequipe.fr/",
            "http://www.lemonde.fr/", "http://www.bbc.co.uk/",
            "http://storm.apache.org/", "http://digitalpebble.com/" };

    builder.setSpout("spout", new MemorySpout(testURLs));

    builder.setBolt("partitioner", new URLPartitionerBolt())
            .shuffleGrouping("spout");

    builder.setBolt("fetch", new FetcherBolt())
            .fieldsGrouping("partitioner", new Fields("key"));

    builder.setBolt("sitemap", new SiteMapParserBolt())
            .localOrShuffleGrouping("fetch");

    builder.setBolt("feeds", new FeedParserBolt())
            .localOrShuffleGrouping("sitemap");

    builder.setBolt("parse", new JSoupParserBolt())
            .localOrShuffleGrouping("feeds");

    builder.setBolt("index", new StdOutIndexer())
            .localOrShuffleGrouping("parse");

    Fields furl = new Fields("url");

    // can also use MemoryStatusUpdater for simple recursive crawls
    builder.setBolt("status", new StdOutStatusUpdater())
            .fieldsGrouping("fetch", Constants.StatusStreamName, furl)
            .fieldsGrouping("sitemap", Constants.StatusStreamName, furl)
            .fieldsGrouping("feeds", Constants.StatusStreamName, furl)
            .fieldsGrouping("parse", Constants.StatusStreamName, furl)
            .fieldsGrouping("index", Constants.StatusStreamName, furl);

    return submit("crawl", conf, builder);
}
 
Example #6
Source File: App.java    From java-study with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args)  {
		//定义一个拓扑
		TopologyBuilder builder=new TopologyBuilder();
		//设置两个Executeor(线程),默认一个
		builder.setSpout(test_spout, new TestSpout(),2);
		//shuffleGrouping:表示是随机分组
		//设置两个Executeor(线程),和两个task
		builder.setBolt(test_bolt, new TestBolt(),2).setNumTasks(2).shuffleGrouping(test_spout);
		//fieldsGrouping:表示是按字段分组
		//设置两个Executeor(线程),和两个task
		builder.setBolt(test2_bolt, new Test2Bolt(),2).setNumTasks(2).fieldsGrouping(test_bolt, new Fields("count"));
		Config conf = new Config();
		conf.put("test", "test");
		try{
		  //运行拓扑
	       if(args !=null&&args.length>0){ //有参数时,表示向集群提交作业,并把第一个参数当做topology名称
	       	 System.out.println("运行远程模式");
			 StormSubmitter.submitTopology(args[0], conf, builder.createTopology());
	      } else{//没有参数时,本地提交
	        //启动本地模式
	     	System.out.println("运行本地模式");
	        LocalCluster cluster = new LocalCluster();
	        cluster.submitTopology("Word-counts" ,conf,  builder.createTopology() );
	        Thread.sleep(20000);
//	        //关闭本地集群
	        cluster.shutdown();
	      }
		}catch (Exception e){
			e.printStackTrace();
		}
	}
 
Example #7
Source File: TridentReach.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
public static StormTopology buildTopology() {
    TridentTopology topology = new TridentTopology();
    TridentState urlToTweeters = topology.newStaticState(new StaticSingleKeyMapState.Factory(TWEETERS_DB));
    TridentState tweetersToFollowers = topology.newStaticState(new StaticSingleKeyMapState.Factory(FOLLOWERS_DB));


    topology.newDRPCStream("reach").stateQuery(urlToTweeters, new Fields("args"), new MapGet(), new Fields(
        "tweeters")).each(new Fields("tweeters"), new ExpandList(), new Fields("tweeter")).shuffle().stateQuery(
        tweetersToFollowers, new Fields("tweeter"), new MapGet(), new Fields("followers")).each(new Fields("followers"),
                                                                                                new ExpandList(),
                                                                                                new Fields("follower"))
            .groupBy(new Fields("follower")).aggregate(new One(), new Fields(
        "one")).aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    return topology.build();
}
 
Example #8
Source File: ThresholdingEngineAlarmTest.java    From monasca-thresh with Apache License 2.0 5 votes vote down vote up
@BeforeMethod
public void beforeMethod() throws Exception {
  // Fixtures
  alarmDefinitionDAO = mock(AlarmDefinitionDAO.class);

  // Mocks
  alarmDAO = new MockAlarmDAO();

  // Bindings
  Injector.reset();
  Injector.registerModules(new AbstractModule() {
    protected void configure() {
      bind(AlarmDAO.class).toInstance(alarmDAO);
      bind(AlarmDefinitionDAO.class).toInstance(alarmDefinitionDAO);
    }
  });

  // Config
  ThresholdingConfiguration threshConfig = new ThresholdingConfiguration();
  threshConfig.alarmDelay = 1;
  threshConfig.sporadicMetricNamespaces = new HashSet<String>();
  Serialization.registerTarget(KafkaProducerConfiguration.class);

  threshConfig.kafkaProducerConfig =
      Serialization
          .fromJson("{\"KafkaProducerConfiguration\":{\"topic\":\"alarm-state-transitions\",\"metadataBrokerList\":\"192.168.10.10:9092\",\"requestRequiredAcks\":1,\"requestTimeoutMs\":10000,\"producerType\":\"sync\",\"serializerClass\":\"kafka.serializer.StringEncoder\",\"keySerializerClass\":\"\",\"partitionerClass\":\"\",\"compressionCodec\":\"none\",\"compressedTopics\":\"\",\"messageSendMaxRetries\":3,\"retryBackoffMs\":100,\"topicMetadataRefreshIntervalMs\":600000,\"queueBufferingMaxMs\":5000,\"queueBufferingMaxMessages\":10000,\"queueEnqueueTimeoutMs\":-1,\"batchNumMessages\":200,\"sendBufferBytes\":102400,\"clientId\":\"Threshold_Engine\"}}");
  Config stormConfig = new Config();
  stormConfig.setMaxTaskParallelism(1);
  metricSpout = new FeederSpout(new Fields(MetricSpout.FIELDS));
  eventSpout = new FeederSpout(new Fields("event"));
  alarmEventForwarder = mock(AlarmEventForwarder.class);
  Injector
      .registerModules(new TopologyModule(threshConfig, stormConfig, metricSpout, eventSpout));
  Injector.registerModules(new ProducerModule(alarmEventForwarder));

  // Evaluate alarm stats every 5 seconds
  System.setProperty(MetricAggregationBolt.TICK_TUPLE_SECONDS_KEY, "5");

  startTopology();
}
 
Example #9
Source File: GeneralTopologyContext.java    From incubator-heron with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
public GeneralTopologyContext(StormTopology topology, Map stormConf,
                              Map<Integer, String> taskToComponent,
                              Map<String, List<Integer>> componentToSortedTasks,
                              Map<String, Map<String, Fields>> componentToStreamToFields,
                              String stormId) {
  throw new RuntimeException("GeneralTopologyContext should never be initiated this way");
}
 
Example #10
Source File: NotificationsTestBolt.java    From streamline with Apache License 2.0 5 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    if (!consoleNotificationStream.isEmpty()) {
        declarer.declareStream(consoleNotificationStream, new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
    if (!emailNotificationStream.isEmpty()) {
        declarer.declareStream(emailNotificationStream, new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
}
 
Example #11
Source File: NormalizationBolt.java    From streamline with Apache License 2.0 5 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    if (normalizationProcessor == null) {
        throw new RuntimeException("normalizationProcessor cannot be null");
    }
    for (Stream stream : normalizationProcessor.getOutputStreams()) {
        declarer.declareStream(stream.getId(), new Fields(StreamlineEvent.STREAMLINE_EVENT));
    }
}
 
Example #12
Source File: HdfsTextOutputFormat.java    From streamline with Apache License 2.0 5 votes vote down vote up
/**
 * Only output the specified fields.
 *
 * @param commaSeparatedFields  Names of output fields, in the order they should appear in file
 * @return
 */
public HdfsTextOutputFormat withFields(String commaSeparatedFields){
    String[] rawFields = commaSeparatedFields.split(",");
    String[] trimmedFields = new String[rawFields.length];
    for (int i = 0; i < rawFields.length; i++)   {
        trimmedFields[i] = rawFields[i].trim();
    }

    fields = new Fields(trimmedFields);
    return this;
}
 
Example #13
Source File: SpringRPCRequestTest.java    From breeze with Apache License 2.0 5 votes vote down vote up
@Test
public void oneArgument() {
	SpringRPCRequest subject = new SpringRPCRequest("fn(x)");
	OutputFieldsDeclarer declarerMock = mock(OutputFieldsDeclarer.class);
	subject.declareOutputFields(declarerMock);
	ArgumentCaptor<Fields> fieldsCaptor = ArgumentCaptor.forClass(Fields.class);
	verify(declarerMock).declareStream(eq("default"), fieldsCaptor.capture());
	assertEquals(asList("x", "fn-rpc-ctx"), fieldsCaptor.getValue().toList());
}
 
Example #14
Source File: NiFiSpout.java    From localization_nifi with Apache License 2.0 5 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
    final List<String> fieldNames = new ArrayList<>();
    fieldNames.add(NIFI_DATA_PACKET);
    fieldNames.addAll(attributeNames);
    outputFieldsDeclarer.declare(new Fields(fieldNames));
}
 
Example #15
Source File: TridentMinMaxOfVehiclesTopology.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a topology which demonstrates min/max operations on tuples of stream which contain vehicle and driver fields
 * with values {@link TridentMinMaxOfVehiclesTopology.Vehicle} and {@link TridentMinMaxOfVehiclesTopology.Driver} respectively.
 */
public static StormTopology buildVehiclesTopology() {
    Fields driverField = new Fields(Driver.FIELD_NAME);
    Fields vehicleField = new Fields(Vehicle.FIELD_NAME);
    Fields allFields = new Fields(Vehicle.FIELD_NAME, Driver.FIELD_NAME);

    FixedBatchSpout spout = new FixedBatchSpout(allFields, 10, Vehicle.generateVehicles(20));
    spout.setCycle(true);

    TridentTopology topology = new TridentTopology();
    Stream vehiclesStream = topology.newStream("spout1", spout).
        each(allFields, new Debug("##### vehicles"));

    Stream slowVehiclesStream =
        vehiclesStream
            .min(new SpeedComparator())
            .each(vehicleField, new Debug("#### slowest vehicle"));

    Stream slowDriversStream =
        slowVehiclesStream
            .project(driverField)
            .each(driverField, new Debug("##### slowest driver"));

    vehiclesStream
        .max(new SpeedComparator())
        .each(vehicleField, new Debug("#### fastest vehicle"))
        .project(driverField)
        .each(driverField, new Debug("##### fastest driver"));

    vehiclesStream
        .minBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
        each(vehicleField, new Debug("#### least efficient vehicle"));

    vehiclesStream
        .maxBy(Vehicle.FIELD_NAME, new EfficiencyComparator()).
        each(vehicleField, new Debug("#### most efficient vehicle"));

    return topology.build();
}
 
Example #16
Source File: IntermediateRankingsBoltTest.java    From storm-net-adapter with Apache License 2.0 5 votes vote down vote up
@Test
public void shouldDeclareOutputFields() {
    // given
    OutputFieldsDeclarer declarer = mock(OutputFieldsDeclarer.class);
    IntermediateRankingsBolt bolt = new IntermediateRankingsBolt();

    // when
    bolt.declareOutputFields(declarer);

    // then
    verify(declarer, times(1)).declare(any(Fields.class));
}
 
Example #17
Source File: QuerySpout.java    From bullet-storm with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declareStream(QUERY_STREAM, new Fields(ID_FIELD, QUERY_FIELD, METADATA_FIELD));
    declarer.declareStream(METADATA_STREAM, new Fields(ID_FIELD, METADATA_FIELD));
}
 
Example #18
Source File: SentenceWordCountTopology.java    From incubator-heron with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
  outputFieldsDeclarer.declare(new Fields("sentence"));
}
 
Example #19
Source File: TumblingWindowTopology.java    From twister2 with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declare(new Fields("word"));
}
 
Example #20
Source File: BulkMessageWriterBolt.java    From metron with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declareStream(Constants.ERROR_STREAM, new Fields("message"));
}
 
Example #21
Source File: SentenceWordCountTopology.java    From twister2 with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
  outputFieldsDeclarer.declare(new Fields("sentence"));
}
 
Example #22
Source File: TestWindowedQueryBolt.java    From streamline with Apache License 2.0 4 votes vote down vote up
public Fields getComponentOutputFields(String componentId, String streamId) {
    return fields;
}
 
Example #23
Source File: DispatcherBout.java    From DBus with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("subPackage", "currentOffset"));
}
 
Example #24
Source File: KafkaConsumerSpout.java    From DBus with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("record"));
}
 
Example #25
Source File: Twister2Bolt.java    From twister2 with Apache License 2.0 4 votes vote down vote up
public void addInboundFieldsForEdge(String edge, Fields fields) {
  LOG.info(String.format("[Storm-Bolt : %s] Adding inbound fields for edge %s "
      + "with fields %s", id, edge, fields));
  this.inboundEdgeToFieldsMap.put(edge, fields);
}
 
Example #26
Source File: TestIBasicPrintBolt.java    From incubator-heron with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
  declarer.declare(new Fields("ibasic-print"));
}
 
Example #27
Source File: RawMessageUtilTest.java    From metron with Apache License 2.0 4 votes vote down vote up
private void checkEnvelopeMetadata(RawMessage m) {
  assertEquals("real_original_string", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + "." + Constants.Fields.ORIGINAL.getName()));
  assertEquals("enveloped_metadata_val_1", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + ".enveloped_metadata_field_1"));
  assertEquals("enveloped_metadata_val_2", m.getMetadata().get(MetadataUtil.METADATA_PREFIX + ".enveloped_metadata_field_2"));
}
 
Example #28
Source File: JSONScheme.java    From nightwatch with GNU Lesser General Public License v3.0 4 votes vote down vote up
@Override
public Fields getOutputFields() {
    if(outputFields.size() == 0) return null;
    return new Fields(outputFields);
}
 
Example #29
Source File: OutputFieldsGetter.java    From incubator-heron with Apache License 2.0 4 votes vote down vote up
public void declareStream(String streamId, boolean direct, Fields fields) {
  delegate.declareStream(streamId, direct, fields.getDelegate());
}
 
Example #30
Source File: RedirectionBolt.java    From storm-crawler with Apache License 2.0 4 votes vote down vote up
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
    declarer.declare(new Fields("url", "content", "metadata", "text"));
    declarer.declareStream("tika", new Fields("url", "content", "metadata",
            "text"));
}