Java Code Examples for backtype.storm.utils.Utils#getInt()

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _rand = new Random(Utils.secureRandomLong());
    _state = TransactionalState.newCoordinatorState(conf, (String) conf.get(Config.TOPOLOGY_TRANSACTIONAL_ID), _spout.getComponentConfiguration());
    _coordinatorState = new RotatingTransactionalState(_state, META_DIR, true);
    _collector = collector;
    _coordinator = _spout.getCoordinator(conf, context);
    _currTransaction = getStoredCurrTransaction(_state);
    Object active = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (active == null) {
        _maxTransactionActive = 1;
    } else {
        _maxTransactionActive = Utils.getInt(active);
    }
    _initializer = new StateInitializer();
}
 

public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 

public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 

@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _clients = new ArrayList<>();
    if (_local_drpc_id == null) {
        _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>());
        _futures = new LinkedList<>();

        int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
        int index = context.getThisTaskIndex();

        int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
        List<String> servers = NetWorkUtils.host2Ip((List<String>) conf.get(Config.DRPC_SERVERS));

        if (servers == null || servers.isEmpty()) {
            throw new RuntimeException("No DRPC servers configured for topology");
        }

        if (numTasks < servers.size()) {
            for (String s : servers) {
                _futures.add(_backround.submit(new Adder(s, port, conf)));
            }
        } else {
            int i = index % servers.size();
            _futures.add(_backround.submit(new Adder(servers.get(i), port, conf)));
        }
    }

}
 

public Emitter(Map conf) {
    Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (c == null)
        _maxSpoutPending = 1;
    else
        _maxSpoutPending = Utils.getInt(c);
}
 

public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " +
                "StormSubmitter knows which jar to upload.");
    }

    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 

protected List<AsyncLoopThread> setSerializeThreads() {
    WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology);
    int tasksNum = shutdownTasks.size();
    double workerRatio = ConfigExtension.getWorkerSerializeThreadRatio(stormConf);
    int workerSerialThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum));
    if (workerSerialThreadNum > 0 && tasksNum > 0) {
        double average = tasksNum / (double) workerSerialThreadNum;
        for (int i = 0; i < workerSerialThreadNum; i++) {
            int startRunTaskIndex = Utils.getInt(Math.rint(average * i));
            serializeThreads.add(new AsyncLoopThread(new WorkerSerializeRunnable(
                    shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i)));
        }
    }
    return serializeThreads;
}
 

protected List<AsyncLoopThread> setDeserializeThreads() {
    WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology);
    int tasksNum = shutdownTasks.size();
    double workerRatio = ConfigExtension.getWorkerDeserializeThreadRatio(stormConf);
    int workerDeserThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum));
    if (workerDeserThreadNum > 0 && tasksNum > 0) {
        double average = tasksNum / (double) workerDeserThreadNum;
        for (int i = 0; i < workerDeserThreadNum; i++) {
            int startRunTaskIndex = Utils.getInt(Math.rint(average * i));
            deserializeThreads.add(new AsyncLoopThread(new WorkerDeserializeRunnable(
                    shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i)));
        }
    }
    return deserializeThreads;
}
 

@SuppressWarnings("rawtypes")
NettyClientAsync(Map conf, ChannelFactory factory, String host, int port, ReconnectRunnable reconnector, final Set<Integer> sourceTasks, final Set<Integer> targetTasks) {
    super(conf, factory, host, port, reconnector);

    clientChannelFactory = factory;
    initFlowCtrl(conf, sourceTasks, targetTasks);
    
    flushCheckInterval = Utils.getInt(conf.get(Config.STORM_NETTY_FLUSH_CHECK_INTERVAL_MS), 5);
    flusher = new NettyClientFlush(flushCheckInterval);
    flusher.start();

    start();
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1);
    isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false);
}
 

public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    this.count = 0l;
    this.isAckRequired = Utils.getBoolean(conf.get("is.ack.required"), false);
    this.sendNumEachTime = Utils.getInt(conf.get("send.num.each.time"), 1);
    this.startTimeStamp = System.currentTimeMillis();
}
 

private void checkTopologyTimeout(QueueService.Iface queueClient, Map map) {
    int topologyTimeout = Utils.getInt(map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    GetQueueInfoRequest getQueueInfoRequest = new GetQueueInfoRequest(emqConfig.queueName);
    GetQueueInfoResponse response = null;
    try {
        response = queueClient.getQueueInfo(getQueueInfoRequest);
    } catch (TException e) {
        throw new RuntimeException("Get EMQ queue info failed: " + e);
    }

    int emqInvisibleTime = response.getQueueAttribute().getInvisibilitySeconds();
    if (emqInvisibleTime < topologyTimeout)
        throw new RuntimeException("TOPOLOGY_MESSAGE_TIMEOUT_SECS(" + topologyTimeout +
                "s) must small than EMQ queue invisibilitySeconds(" + emqInvisibleTime + "s)");
}
 

public static Integer getInt(Map conf, String key) {
    Object value = conf.get(key);
    if (value == null) {
        throw new RuntimeException("Please set value for Key: " + key);
    }

    Integer result = Utils.getInt(value);
    if (null == result) {
        throw new IllegalArgumentException("Don't know how to convert null to int");
    }
    return result;
}
 

public Emitter(Map conf) {
    Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (c == null)
        _maxSpoutPending = 1;
    else
        _maxSpoutPending = Utils.getInt(c);
}
 

public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 

public static void test() throws Exception{
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 2);
    
    TopologyBuilder builder = new TopologyBuilder();
    
    
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}
 

@SuppressWarnings("rawtypes")
NettyClient(Map storm_conf, ChannelFactory factory, String host, int port, ReconnectRunnable reconnector) {
    this.stormConf = storm_conf;
    this.factory = factory;
    this.reconnector = reconnector;

    retries = new AtomicInteger(0);
    channelRef = new AtomicReference<>(null);
    beingClosed = new AtomicBoolean(false);
    pendings = new AtomicLong(0);

    nettyConnection = new NettyConnection();
    nettyConnection.setClientPort(NetWorkUtils.ip(), ConfigExtension.getLocalWorkerPort(storm_conf));
    nettyConnection.setServerPort(host, port);

    // Configure
    bufferSize = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE));
    maxRetries = Math.min(30, Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_RETRIES)));
    baseSleepMs = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS));
    maxSleepMs = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS));

    timeoutMs = ConfigExtension.getNettyPendingBufferTimeout(storm_conf);
    MAX_SEND_PENDING = (int) ConfigExtension.getNettyMaxSendPending(storm_conf);

    this.messageBatchSize = Utils.getInt(storm_conf.get(Config.STORM_NETTY_MESSAGE_BATCH_SIZE), 262144);
    this.messageBuffer = new MessageBuffer(messageBatchSize);

    this.writeLock = new Object();

    // Start the connection attempt.
    remoteAddr = new InetSocketAddress(host, port);
    name = remoteAddr.toString();
    connectMyself = isConnectMyself(stormConf, host, port);

    address = JStormServerUtils.getName(host, port);

    this.enableNettyMetrics = MetricUtils.isEnableNettyMetrics(storm_conf);
    NettyMetricInstance.register();
    LOG.info("* enable netty metrics: {}", this.enableNettyMetrics);
    if (!connectMyself) {
        registerMetrics();
    }
    closingChannel = new HashSet<Channel>();

    BATCH_THRESHOLD_WARN = ConfigExtension.getNettyBufferThresholdSize(stormConf);
}
 

public static int getInt(Map map, Object key, int def) {
  return Utils.getInt(Utils.get(map, key, def));
}
 

/**
 * Invoked once immediately after construction
 * 
 * @param storm_conf Storm configuration
 */
@Override
public void prepare(Map storm_conf) {
    int timeout = Utils.getInt(storm_conf.get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS));
    cachedGroups = new TimeCacheMap<String, Set<String>>(timeout);
}
 

@Override
  public StormTopology getTopology(Config config) {

    Object sObj = config.get(SERVER);
    if (null == sObj) {
      throw new IllegalArgumentException("must set a drpc server");
    }
    server = (String) sObj;
    config.put(Config.DRPC_SERVERS, Lists.newArrayList(server));

    Object pObj = config.get(PORT);
    if (null == pObj) {
      throw new IllegalArgumentException("must set a drpc port");
    }
    port = Utils.getInt(pObj);
    config.put(Config.DRPC_PORT, port);

    LOG.info("drpc server: " + server + "; drpc port: " + port);

    final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int pageNum = BenchmarkUtils.getInt(config, PAGE_NUM, DEFAULT_PAGE_BOLT_NUM);
    final int viewNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_VIEW_BOLT_NUM);
    final int userNum = BenchmarkUtils.getInt(config, USER_NUM, DEFAULT_USER_BOLT_NUM);
    final int followerNum = BenchmarkUtils.getInt(config, FOLLOWER_NUM, DEFAULT_FOLLOWER_BOLT_NUM);

    spout = new TransactionalTridentKafkaSpout(
            KafkaUtils.getTridentKafkaConfig(config, new SchemeAsMultiScheme(new StringScheme())));

    TridentTopology trident = new TridentTopology();
    TridentState urlToUsers =
            trident.newStream("drpc", spout).parallelismHint(spoutNum).shuffle()
            .each(new Fields(StringScheme.STRING_SCHEME_KEY), new Extract(Arrays.asList(Item.URL, Item.USER)),
                    new Fields("url", "user")).parallelismHint(pageNum)
            .groupBy(new Fields("url"))
            .persistentAggregate(new MemoryMapState.Factory(), new Fields("url", "user"), new Distinct(), new Fields("user_set"))
            .parallelismHint(viewNum);
/** debug
 *  1. this proves that the aggregated result has successfully persisted
    urlToUsers.newValuesStream()
            .each(new Fields("url", "user_set"), new Print("(url, user_set)"), new Fields("url2", "user_set2"));
 */
    PageViewGenerator generator = new PageViewGenerator();
    TridentState userToFollowers = trident.newStaticState(new StaticSingleKeyMapState.Factory(generator.genFollowersDB()));
/** debug
  * 2. this proves that MemoryMapState could be read correctly
   trident.newStream("urlToUsers", new PageViewSpout(false))
            .each(new Fields("page_view"), new Extract(Arrays.asList(Item.URL)), new Fields("url"))
            .each(new Fields("url"), new Print("url"), new Fields("url2"))
            .groupBy(new Fields("url2"))
            .stateQuery(urlToUsers, new Fields("url2"),  new MapGet(), new Fields("users"))
            .each(new Fields("users"), new Print("users"), new Fields("users2"));
*/
/** debug
 *  3. this proves that StaticSingleKeyMapState could be read correctly
    trident.newStream("userToFollowers", new PageViewSpout(false))
            .each(new Fields("page_view"), new Extract(Arrays.asList(Item.USER)), new Fields("user"))
            .each(new Fields("user"), new Print("user"), new Fields("user2"))
            .stateQuery(userToFollowers, new Fields("user2"), new MapGet(), new Fields("followers"))
            .each(new Fields("followers"), new Print("followers"), new Fields("followers2"));
 */
    trident.newDRPCStream(FUNCTION, null)
            .each(new Fields("args"), new Print("args"), new Fields("url"))
            .groupBy(new Fields("url"))
            .stateQuery(urlToUsers, new Fields("url"), new MapGet(), new Fields("users"))
            .each(new Fields("users"), new Expand(), new Fields("user")).parallelismHint(userNum)
            .groupBy(new Fields("user"))
            .stateQuery(userToFollowers, new Fields("user"), new MapGet(), new Fields("followers"))
            .each(new Fields("followers"), new Expand(), new Fields("follower")).parallelismHint(followerNum)
            .groupBy(new Fields("follower"))
            .aggregate(new One(), new Fields("one"))
            .aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    return trident.build();
  }