Java Code Examples for org.apache.storm.Config#putAll()

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}
 

public static Config submitTopology(ILocalCluster stormCluster, String topologyName,
                                    StormTopology stormTopology) throws Exception {
    Config stormConf = new Config();
    stormConf.putAll(Utils.readDefaultConfig());
    stormConf.put("storm.cluster.mode", "local");
    stormConf.setDebug(true);
    stormConf.setMaxTaskParallelism(3);
    stormConf.put(Config.STORM_TOPOLOGY_SUBMISSION_NOTIFIER_PLUGIN,
            org.apache.atlas.storm.hook.StormAtlasHook.class.getName());

    stormCluster.submitTopology(topologyName, stormConf, stormTopology);

    Thread.sleep(10000);
    return stormConf;
}
 

public static Optional<Config> getConfig(CommandLine cli, Config config) {
  if(EXTRA_OPTIONS.has(cli)) {
    Map<String, Object> extraOptions = readJSONMapFromFile(new File(EXTRA_OPTIONS.get(cli)));
    config.putAll(extraOptions);
  }
  for(ParserOptions option : ParserOptions.values()) {
    config = option.configHandler.apply(new Arg(config, option.get(cli)));
  }
  return config.isEmpty()?Optional.empty():Optional.of(config);
}
 

@Override
public void configure(Map stormConf, JsonNode filterParams) {
    Config conf = new Config();
    conf.putAll(stormConf);
    factory = new ProtocolFactory(conf);
    robots = new HttpRobotRulesParser(conf);

    JsonNode node = filterParams.get("fromCacheOnly");
    if (node != null && node.isBoolean()) {
        fromCacheOnly = node.booleanValue();
    }
}
 

public static Config getConfig(String[] args) {
    Config ret = new Config();
    if (args == null || args.length == 0) {
        return ret;
    }
    
    if (StringUtils.isBlank(args[0])) {
        return ret;
    }
    
    Map conf = LoadConf(args[0]);
    ret.putAll(conf);
    return ret;
}
 

/**
 * This function can be used to wire up the source of the records to Bullet. The name of the last component in your
 * topology and the {@link TopologyBuilder} used to create your topology should be provided. That topology
 * will be wired up with Bullet reading from your component that produces the {@link com.yahoo.bullet.record.BulletRecord}.
 *
 * @param config The non-null, validated {@link BulletStormConfig} that contains the necessary configuration.
 * @param recordComponent The non-null name of the component used in your topology that is the source of records for Bullet.
 * @param builder The non-null {@link TopologyBuilder} that was used to create your topology.
 * @throws Exception if there were issues creating the topology.
 */
public static void submit(BulletStormConfig config, String recordComponent, TopologyBuilder builder) throws Exception {
    Objects.requireNonNull(config);
    Objects.requireNonNull(recordComponent);
    Objects.requireNonNull(builder);

    String name = config.getAs(BulletStormConfig.TOPOLOGY_NAME, String.class);

    Number querySpoutParallelism = config.getAs(BulletStormConfig.QUERY_SPOUT_PARALLELISM, Number.class);
    Number querySpoutCPULoad = config.getAs(BulletStormConfig.QUERY_SPOUT_CPU_LOAD, Number.class);
    Number querySpoutMemoryOnHeapLoad = config.getAs(BulletStormConfig.QUERY_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number querySpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.QUERY_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class);

    // Tick parallelism must be 1 otherwise multiple ticks will get delivered to a component
    Number tickSpoutParallelism = BulletStormConfig.TICK_SPOUT_PARALLELISM;
    Number tickSpoutCPULoad = config.getAs(BulletStormConfig.TICK_SPOUT_CPU_LOAD, Number.class);
    Number tickSpoutMemoryOnheapLoad = config.getAs(BulletStormConfig.TICK_SPOUT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number tickSpoutMemoryOffHeapLoad = config.getAs(BulletStormConfig.TICK_SPOUT_MEMORY_OFF_HEAP_LOAD, Number.class);

    Number filterBoltParallelism = config.getAs(BulletStormConfig.FILTER_BOLT_PARALLELISM, Number.class);
    Number filterBoltCPULoad = config.getAs(BulletStormConfig.FILTER_BOLT_CPU_LOAD, Number.class);
    Number filterBoltMemoryOnheapLoad = config.getAs(BulletStormConfig.FILTER_BOLT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number filterBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.FILTER_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class);

    Number joinBoltParallelism = config.getAs(BulletStormConfig.JOIN_BOLT_PARALLELISM, Number.class);
    Number joinBoltCPULoad = config.getAs(BulletStormConfig.JOIN_BOLT_CPU_LOAD, Number.class);
    Number joinBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.JOIN_BOLT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number joinBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.JOIN_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class);

    Number resultBoltParallelism = config.getAs(BulletStormConfig.RESULT_BOLT_PARALLELISM, Number.class);
    Number resultBoltCPULoad = config.getAs(BulletStormConfig.RESULT_BOLT_CPU_LOAD, Number.class);
    Number resultBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.RESULT_BOLT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number resultBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.RESULT_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class);

    Number loopBoltParallelism = config.getAs(BulletStormConfig.LOOP_BOLT_PARALLELISM, Number.class);
    Number loopBoltCPULoad = config.getAs(BulletStormConfig.LOOP_BOLT_CPU_LOAD, Number.class);
    Number loopBoltMemoryOnHeapLoad = config.getAs(BulletStormConfig.LOOP_BOLT_MEMORY_ON_HEAP_LOAD, Number.class);
    Number loopBoltMemoryOffHeapLoad = config.getAs(BulletStormConfig.LOOP_BOLT_MEMORY_OFF_HEAP_LOAD, Number.class);

    builder.setSpout(QUERY_COMPONENT, new QuerySpout(config), querySpoutParallelism)
           .setCPULoad(querySpoutCPULoad).setMemoryLoad(querySpoutMemoryOnHeapLoad, querySpoutMemoryOffHeapLoad);

    builder.setSpout(TICK_COMPONENT, new TickSpout(config), tickSpoutParallelism)
           .setCPULoad(tickSpoutCPULoad).setMemoryLoad(tickSpoutMemoryOnheapLoad, tickSpoutMemoryOffHeapLoad);

    // Hook in the source of the BulletRecords
    builder.setBolt(FILTER_COMPONENT, new FilterBolt(recordComponent, config), filterBoltParallelism)
           .shuffleGrouping(recordComponent)
           .allGrouping(QUERY_COMPONENT, QUERY_STREAM)
           .allGrouping(QUERY_COMPONENT, METADATA_STREAM)
           .allGrouping(TICK_COMPONENT, TICK_STREAM)
           .setCPULoad(filterBoltCPULoad).setMemoryLoad(filterBoltMemoryOnheapLoad, filterBoltMemoryOffHeapLoad);

    builder.setBolt(JOIN_COMPONENT, new JoinBolt(config), joinBoltParallelism)
           .fieldsGrouping(QUERY_COMPONENT, QUERY_STREAM, new Fields(ID_FIELD))
           .fieldsGrouping(QUERY_COMPONENT, METADATA_STREAM, new Fields(ID_FIELD))
           .fieldsGrouping(FILTER_COMPONENT, DATA_STREAM, new Fields(ID_FIELD))
           .allGrouping(TICK_COMPONENT, TICK_STREAM)
           .setCPULoad(joinBoltCPULoad).setMemoryLoad(joinBoltMemoryOnHeapLoad, joinBoltMemoryOffHeapLoad);

    builder.setBolt(TopologyConstants.RESULT_COMPONENT, new ResultBolt(config), resultBoltParallelism)
           .shuffleGrouping(JOIN_COMPONENT, RESULT_STREAM)
           .setCPULoad(resultBoltCPULoad).setMemoryLoad(resultBoltMemoryOnHeapLoad, resultBoltMemoryOffHeapLoad);

    // Hook in the Loop Bolt only if windowing is enabled
    boolean isWindowingDisabled = config.getAs(BulletConfig.WINDOW_DISABLE, Boolean.class);
    if (isWindowingDisabled) {
        log.info("Windowing is disabled. Skipping hooking in the Loop Bolt...");
    } else {
        builder.setBolt(LOOP_COMPONENT, new LoopBolt(config), loopBoltParallelism)
               .shuffleGrouping(JOIN_COMPONENT, FEEDBACK_STREAM)
               .setCPULoad(loopBoltCPULoad).setMemoryLoad(loopBoltMemoryOnHeapLoad, loopBoltMemoryOffHeapLoad);
    }

    Config stormConfig = new Config();

    // Metrics
    Boolean enableMetrics = (Boolean) config.get(BulletStormConfig.TOPOLOGY_METRICS_ENABLE);
    if (enableMetrics) {
        List<String> classNames = config.getAs(BulletStormConfig.TOPOLOGY_METRICS_CLASSES, List.class);
        classNames.forEach(className -> ReflectionUtils.registerMetricsConsumer(className, stormConfig, config));
    }

    // Put the rest of the other possible custom Storm settings without checking their types
    stormConfig.putAll(config.getCustomStormSettings());

    StormSubmitter.submitTopology(name, stormConfig, builder.createTopology());
}
 

public static void prepare() {
    Config conf = new Config();
    conf.putAll(Utils.readStormConfig());
    store = Utils.getClientBlobStore(conf);
}
 

/** Called by extensions of this class **/
protected static void main(AbstractHttpProtocol protocol, String args[])
        throws Exception {
    Config conf = new Config();

    // loads the default configuration file
    Map defaultSCConfig = Utils.findAndReadConfigFile(
            "crawler-default.yaml", false);
    conf.putAll(ConfUtils.extractConfigElement(defaultSCConfig));

    Options options = new Options();
    options.addOption("c", true, "configuration file");

    CommandLineParser parser = new DefaultParser();
    CommandLine cmd = parser.parse(options, args);

    if (cmd.hasOption("c")) {
        String confFile = cmd.getOptionValue("c");
        ConfUtils.loadConf(confFile, conf);
    }

    protocol.configure(conf);

    Set<Runnable> threads = new HashSet<>();

    class Fetchable implements Runnable {
        String url;
        Metadata md;

        Fetchable(String line) {
            StringTabScheme scheme = new StringTabScheme();
            List<Object> tuple = scheme.deserialize(ByteBuffer.wrap(line
                    .getBytes(StandardCharsets.UTF_8)));
            this.url = (String) tuple.get(0);
            this.md = (Metadata) tuple.get(1);
        }

        public void run() {

            StringBuilder stringB = new StringBuilder();
            stringB.append(url).append("\n");

            if (!protocol.skipRobots) {
                BaseRobotRules rules = protocol.getRobotRules(url);
                stringB.append("robots allowed: ")
                        .append(rules.isAllowed(url)).append("\n");
                if (rules instanceof RobotRules) {
                    stringB.append("robots requests: ")
                            .append(((RobotRules) rules)
                                    .getContentLengthFetched().length)
                            .append("\n");
                }
                stringB.append("sitemaps identified: ")
                        .append(rules.getSitemaps().size()).append("\n");
            }

            long start = System.currentTimeMillis();
            ProtocolResponse response;
            try {
                response = protocol.getProtocolOutput(url, md);
                stringB.append(response.getMetadata()).append("\n");
                stringB.append("status code: ")
                        .append(response.getStatusCode()).append("\n");
                stringB.append("content length: ")
                        .append(response.getContent().length).append("\n");
                long timeFetching = System.currentTimeMillis() - start;
                stringB.append("fetched in : ").append(timeFetching)
                        .append(" msec");
                System.out.println(stringB);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                threads.remove(this);
            }
        }
    }

    for (String arg : cmd.getArgs()) {
        Fetchable p = new Fetchable(arg);
        threads.add(p);
        new Thread(p).start();
    }

    while (threads.size() > 0) {
        Thread.sleep(1000);
    }

    protocol.cleanup();
    System.exit(0);
}
 

@SuppressWarnings({ "rawtypes", "unchecked" })
@Override
public void prepare(Map stormConf, TopologyContext context,
        OutputCollector collector) {

    super.prepare(stormConf, context, collector);

    Config conf = new Config();
    conf.putAll(stormConf);

    checkConfiguration(conf);

    LOG.info("[Fetcher #{}] : starting at {}", taskID, Instant.now());

    int metricsTimeBucketSecs = ConfUtils.getInt(conf,
            "fetcher.metrics.time.bucket.secs", 10);

    // Register a "MultiCountMetric" to count different events in this bolt
    // Storm will emit the counts every n seconds to a special bolt via a
    // system stream
    // The data can be accessed by registering a "MetricConsumer" in the
    // topology
    this.eventCounter = context.registerMetric("fetcher_counter",
            new MultiCountMetric(), metricsTimeBucketSecs);

    // create gauges
    context.registerMetric("activethreads", () -> {
        return activeThreads.get();
    }, metricsTimeBucketSecs);

    context.registerMetric("in_queues", () -> {
        return fetchQueues.inQueues.get();
    }, metricsTimeBucketSecs);

    context.registerMetric("num_queues", () -> {
        return fetchQueues.queues.size();
    }, metricsTimeBucketSecs);

    this.averagedMetrics = context.registerMetric("fetcher_average_perdoc",
            new MultiReducedMetric(new MeanReducer()),
            metricsTimeBucketSecs);

    this.perSecMetrics = context.registerMetric("fetcher_average_persec",
            new MultiReducedMetric(new PerSecondReducer()),
            metricsTimeBucketSecs);

    protocolFactory = new ProtocolFactory(conf);

    this.fetchQueues = new FetchItemQueues(conf);

    this.taskID = context.getThisTaskId();

    int threadCount = ConfUtils.getInt(conf, "fetcher.threads.number", 10);
    for (int i = 0; i < threadCount; i++) { // spawn threads
        new FetcherThread(conf, i).start();
    }

    // keep track of the URLs in fetching
    beingFetched = new String[threadCount];
    Arrays.fill(beingFetched, "");

    sitemapsAutoDiscovery = ConfUtils.getBoolean(stormConf,
            SITEMAP_DISCOVERY_PARAM_KEY, false);

    maxNumberURLsInQueues = ConfUtils.getInt(conf,
            "fetcher.max.urls.in.queues", -1);

    /**
     * If set to a valid path e.g. /tmp/fetcher-dump-{port} on a worker
     * node, the content of the queues will be dumped to the logs for
     * debugging. The port number needs to match the one used by the
     * FetcherBolt instance.
     **/
    String debugfiletriggerpattern = ConfUtils.getString(conf,
            "fetcherbolt.queue.debug.filepath");

    if (StringUtils.isNotBlank(debugfiletriggerpattern)) {
        debugfiletrigger = new File(
                debugfiletriggerpattern.replaceAll("\\{port\\}",
                        Integer.toString(context.getThisWorkerPort())));
    }

}