Java Code Examples for backtype.storm.utils.Utils#getInt()

The following examples show how to use backtype.storm.utils.Utils#getInt() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may want to check out the right sidebar which shows the related API usage.
Example 1
Source Project: jstorm   File: PerformanceTestTopology.java    License: Apache License 2.0 6 votes vote down vote up
public static void SetRemoteTopology()
        throws Exception {
    String streamName = (String) conf.get(Config.TOPOLOGY_NAME);
    if (streamName == null) {
        String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
        streamName = className[className.length - 1];
    }
    
    TopologyBuilder builder = new TopologyBuilder();
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int bolt_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_BOLT_PARALLELISM_HINT), 2);
    builder.setSpout("spout", new TestSpout(), spout_Parallelism_hint);
    
    BoltDeclarer boltDeclarer = builder.setBolt("bolt", new TestBolt(), bolt_Parallelism_hint);
    // localFirstGrouping is only for jstorm
    // boltDeclarer.localFirstGrouping(SequenceTopologyDef.SEQUENCE_SPOUT_NAME);
    boltDeclarer.shuffleGrouping("spout");
    // .addConfiguration(Config.TOPOLOGY_TICK_TUPLE_FREQ_SECS, 60);
    
    StormSubmitter.submitTopology(streamName, conf, builder.createTopology());
    
}
 
Example 2
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _rand = new Random(Utils.secureRandomLong());
    _state = TransactionalState.newCoordinatorState(conf, (String) conf.get(Config.TOPOLOGY_TRANSACTIONAL_ID), _spout.getComponentConfiguration());
    _coordinatorState = new RotatingTransactionalState(_state, META_DIR, true);
    _collector = collector;
    _coordinator = _spout.getCoordinator(conf, context);
    _currTransaction = getStoredCurrTransaction(_state);
    Object active = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (active == null) {
        _maxTransactionActive = 1;
    } else {
        _maxTransactionActive = Utils.getInt(active);
    }
    _initializer = new StateInitializer();
}
 
Example 3
Source Project: PoseidonX   File: StormSubmitterForUCar.java    License: Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 
Example 4
Source Project: PoseidonX   File: StormSubmitterForUCar.java    License: Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'storm' client script so that StormSubmitter knows which jar to upload.");
    }

    try {

        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {

    }
}
 
Example 5
public Emitter(Map conf) {
    Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (c == null)
        _maxSpoutPending = 1;
    else
        _maxSpoutPending = Utils.getInt(c);
}
 
Example 6
Source Project: galaxy-sdk-java   File: ConfigHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static Integer getInt(Map conf, String key) {
    Object value = conf.get(key);
    if (value == null) {
        throw new RuntimeException("Please set value for Key: " + key);
    }

    Integer result = Utils.getInt(value);
    if (null == result) {
        throw new IllegalArgumentException("Don't know how to convert null to int");
    }
    return result;
}
 
Example 7
Source Project: galaxy-sdk-java   File: EMQSpout.java    License: Apache License 2.0 5 votes vote down vote up
private void checkTopologyTimeout(QueueService.Iface queueClient, Map map) {
    int topologyTimeout = Utils.getInt(map.get(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS));
    GetQueueInfoRequest getQueueInfoRequest = new GetQueueInfoRequest(emqConfig.queueName);
    GetQueueInfoResponse response = null;
    try {
        response = queueClient.getQueueInfo(getQueueInfoRequest);
    } catch (TException e) {
        throw new RuntimeException("Get EMQ queue info failed: " + e);
    }

    int emqInvisibleTime = response.getQueueAttribute().getInvisibilitySeconds();
    if (emqInvisibleTime < topologyTimeout)
        throw new RuntimeException("TOPOLOGY_MESSAGE_TIMEOUT_SECS(" + topologyTimeout +
                "s) must small than EMQ queue invisibilitySeconds(" + emqInvisibleTime + "s)");
}
 
Example 8
Source Project: jstorm   File: PerformanceTestTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    this.collector = collector;
    this.count = 0l;
    this.isAckRequired = Utils.getBoolean(conf.get("is.ack.required"), false);
    this.sendNumEachTime = Utils.getInt(conf.get("send.num.each.time"), 1);
    this.startTimeStamp = System.currentTimeMillis();
}
 
Example 9
Source Project: jstorm   File: FastWordCountTopology.java    License: Apache License 2.0 5 votes vote down vote up
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _rand = new Random();
    sendingCount = 0;
    startTime = System.currentTimeMillis();
    sendNumPerNexttuple = Utils.getInt(conf.get("send.num.each.time"), 1);
    isStatEnable = Utils.getBoolean(conf.get("is.stat.enable"), false);
}
 
Example 10
Source Project: jstorm   File: NettyClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("rawtypes")
NettyClientAsync(Map conf, ChannelFactory factory, String host, int port, ReconnectRunnable reconnector, final Set<Integer> sourceTasks, final Set<Integer> targetTasks) {
    super(conf, factory, host, port, reconnector);

    clientChannelFactory = factory;
    initFlowCtrl(conf, sourceTasks, targetTasks);
    
    flushCheckInterval = Utils.getInt(conf.get(Config.STORM_NETTY_FLUSH_CHECK_INTERVAL_MS), 5);
    flusher = new NettyClientFlush(flushCheckInterval);
    flusher.start();

    start();
}
 
Example 11
Source Project: jstorm   File: WorkerData.java    License: Apache License 2.0 5 votes vote down vote up
protected List<AsyncLoopThread> setDeserializeThreads() {
    WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology);
    int tasksNum = shutdownTasks.size();
    double workerRatio = ConfigExtension.getWorkerDeserializeThreadRatio(stormConf);
    int workerDeserThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum));
    if (workerDeserThreadNum > 0 && tasksNum > 0) {
        double average = tasksNum / (double) workerDeserThreadNum;
        for (int i = 0; i < workerDeserThreadNum; i++) {
            int startRunTaskIndex = Utils.getInt(Math.rint(average * i));
            deserializeThreads.add(new AsyncLoopThread(new WorkerDeserializeRunnable(
                    shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i)));
        }
    }
    return deserializeThreads;
}
 
Example 12
Source Project: jstorm   File: WorkerData.java    License: Apache License 2.0 5 votes vote down vote up
protected List<AsyncLoopThread> setSerializeThreads() {
    WorkerTopologyContext workerTopologyContext = contextMaker.makeWorkerTopologyContext(sysTopology);
    int tasksNum = shutdownTasks.size();
    double workerRatio = ConfigExtension.getWorkerSerializeThreadRatio(stormConf);
    int workerSerialThreadNum = Utils.getInt(Math.ceil(workerRatio * tasksNum));
    if (workerSerialThreadNum > 0 && tasksNum > 0) {
        double average = tasksNum / (double) workerSerialThreadNum;
        for (int i = 0; i < workerSerialThreadNum; i++) {
            int startRunTaskIndex = Utils.getInt(Math.rint(average * i));
            serializeThreads.add(new AsyncLoopThread(new WorkerSerializeRunnable(
                    shutdownTasks, stormConf, workerTopologyContext, startRunTaskIndex, i)));
        }
    }
    return serializeThreads;
}
 
Example 13
Source Project: jstorm   File: StormSubmitter.java    License: Apache License 2.0 5 votes vote down vote up
public static String submitJar(Map conf, String localJar, String uploadLocation, NimbusClient client) {
    if (localJar == null) {
        throw new RuntimeException("Must submit topologies using the 'jstorm' client script so that " +
                "StormSubmitter knows which jar to upload.");
    }

    try {
        LOG.info("Uploading topology jar " + localJar + " to assigned location: " + uploadLocation);
        int bufferSize = 512 * 1024;
        Object maxBufSizeObject = conf.get(Config.NIMBUS_THRIFT_MAX_BUFFER_SIZE);
        if (maxBufSizeObject != null) {
            bufferSize = Utils.getInt(maxBufSizeObject) / 2;
        }

        BufferFileInputStream is = new BufferFileInputStream(localJar, bufferSize);
        while (true) {
            byte[] toSubmit = is.read();
            if (toSubmit.length == 0)
                break;
            client.getClient().uploadChunk(uploadLocation, ByteBuffer.wrap(toSubmit));
        }
        client.getClient().finishFileUpload(uploadLocation);
        LOG.info("Successfully uploaded topology jar to assigned location: " + uploadLocation);
        return uploadLocation;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}
 
Example 14
Source Project: jstorm   File: MemoryTransactionalSpout.java    License: Apache License 2.0 5 votes vote down vote up
public Emitter(Map conf) {
    Object c = conf.get(Config.TOPOLOGY_MAX_SPOUT_PENDING);
    if (c == null)
        _maxSpoutPending = 1;
    else
        _maxSpoutPending = Utils.getInt(c);
}
 
Example 15
Source Project: jstorm   File: DRPCSpout.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _clients = new ArrayList<>();
    if (_local_drpc_id == null) {
        _backround = new ExtendedThreadPoolExecutor(0, Integer.MAX_VALUE, 60L, TimeUnit.SECONDS,
                new SynchronousQueue<Runnable>());
        _futures = new LinkedList<>();

        int numTasks = context.getComponentTasks(context.getThisComponentId()).size();
        int index = context.getThisTaskIndex();

        int port = Utils.getInt(conf.get(Config.DRPC_INVOCATIONS_PORT));
        List<String> servers = NetWorkUtils.host2Ip((List<String>) conf.get(Config.DRPC_SERVERS));

        if (servers == null || servers.isEmpty()) {
            throw new RuntimeException("No DRPC servers configured for topology");
        }

        if (numTasks < servers.size()) {
            for (String s : servers) {
                _futures.add(_backround.submit(new Adder(s, port, conf)));
            }
        } else {
            int i = index % servers.size();
            _futures.add(_backround.submit(new Adder(servers.get(i), port, conf)));
        }
    }

}
 
Example 16
Source Project: storm-benchmark   File: DRPC.java    License: Apache License 2.0 4 votes vote down vote up
@Override
  public StormTopology getTopology(Config config) {

    Object sObj = config.get(SERVER);
    if (null == sObj) {
      throw new IllegalArgumentException("must set a drpc server");
    }
    server = (String) sObj;
    config.put(Config.DRPC_SERVERS, Lists.newArrayList(server));

    Object pObj = config.get(PORT);
    if (null == pObj) {
      throw new IllegalArgumentException("must set a drpc port");
    }
    port = Utils.getInt(pObj);
    config.put(Config.DRPC_PORT, port);

    LOG.info("drpc server: " + server + "; drpc port: " + port);

    final int spoutNum = BenchmarkUtils.getInt(config, SPOUT_NUM, DEFAULT_SPOUT_NUM);
    final int pageNum = BenchmarkUtils.getInt(config, PAGE_NUM, DEFAULT_PAGE_BOLT_NUM);
    final int viewNum = BenchmarkUtils.getInt(config, VIEW_NUM, DEFAULT_VIEW_BOLT_NUM);
    final int userNum = BenchmarkUtils.getInt(config, USER_NUM, DEFAULT_USER_BOLT_NUM);
    final int followerNum = BenchmarkUtils.getInt(config, FOLLOWER_NUM, DEFAULT_FOLLOWER_BOLT_NUM);

    spout = new TransactionalTridentKafkaSpout(
            KafkaUtils.getTridentKafkaConfig(config, new SchemeAsMultiScheme(new StringScheme())));

    TridentTopology trident = new TridentTopology();
    TridentState urlToUsers =
            trident.newStream("drpc", spout).parallelismHint(spoutNum).shuffle()
            .each(new Fields(StringScheme.STRING_SCHEME_KEY), new Extract(Arrays.asList(Item.URL, Item.USER)),
                    new Fields("url", "user")).parallelismHint(pageNum)
            .groupBy(new Fields("url"))
            .persistentAggregate(new MemoryMapState.Factory(), new Fields("url", "user"), new Distinct(), new Fields("user_set"))
            .parallelismHint(viewNum);
/** debug
 *  1. this proves that the aggregated result has successfully persisted
    urlToUsers.newValuesStream()
            .each(new Fields("url", "user_set"), new Print("(url, user_set)"), new Fields("url2", "user_set2"));
 */
    PageViewGenerator generator = new PageViewGenerator();
    TridentState userToFollowers = trident.newStaticState(new StaticSingleKeyMapState.Factory(generator.genFollowersDB()));
/** debug
  * 2. this proves that MemoryMapState could be read correctly
   trident.newStream("urlToUsers", new PageViewSpout(false))
            .each(new Fields("page_view"), new Extract(Arrays.asList(Item.URL)), new Fields("url"))
            .each(new Fields("url"), new Print("url"), new Fields("url2"))
            .groupBy(new Fields("url2"))
            .stateQuery(urlToUsers, new Fields("url2"),  new MapGet(), new Fields("users"))
            .each(new Fields("users"), new Print("users"), new Fields("users2"));
*/
/** debug
 *  3. this proves that StaticSingleKeyMapState could be read correctly
    trident.newStream("userToFollowers", new PageViewSpout(false))
            .each(new Fields("page_view"), new Extract(Arrays.asList(Item.USER)), new Fields("user"))
            .each(new Fields("user"), new Print("user"), new Fields("user2"))
            .stateQuery(userToFollowers, new Fields("user2"), new MapGet(), new Fields("followers"))
            .each(new Fields("followers"), new Print("followers"), new Fields("followers2"));
 */
    trident.newDRPCStream(FUNCTION, null)
            .each(new Fields("args"), new Print("args"), new Fields("url"))
            .groupBy(new Fields("url"))
            .stateQuery(urlToUsers, new Fields("url"), new MapGet(), new Fields("users"))
            .each(new Fields("users"), new Expand(), new Fields("user")).parallelismHint(userNum)
            .groupBy(new Fields("user"))
            .stateQuery(userToFollowers, new Fields("user"), new MapGet(), new Fields("followers"))
            .each(new Fields("followers"), new Expand(), new Fields("follower")).parallelismHint(followerNum)
            .groupBy(new Fields("follower"))
            .aggregate(new One(), new Fields("one"))
            .aggregate(new Fields("one"), new Sum(), new Fields("reach"));
    return trident.build();
  }
 
Example 17
Source Project: storm-benchmark   File: BenchmarkUtils.java    License: Apache License 2.0 4 votes vote down vote up
public static int getInt(Map map, Object key, int def) {
  return Utils.getInt(Utils.get(map, key, def));
}
 
Example 18
Source Project: jstorm   File: FastWordCountTopology.java    License: Apache License 2.0 4 votes vote down vote up
public static void test() throws Exception{
    
    int spout_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPOUT_PARALLELISM_HINT), 1);
    int split_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_SPLIT_PARALLELISM_HINT), 1);
    int count_Parallelism_hint = Utils.getInt(conf.get(TOPOLOGY_COUNT_PARALLELISM_HINT), 2);
    
    TopologyBuilder builder = new TopologyBuilder();
    
    
    builder.setSpout("spout", new FastRandomSentenceSpout(), spout_Parallelism_hint);
    builder.setBolt("split", new SplitSentence(), split_Parallelism_hint).shuffleGrouping("spout");
    builder.setBolt("count", new WordCount(), count_Parallelism_hint).fieldsGrouping("split", new Fields("word"));
    
    String[] className = Thread.currentThread().getStackTrace()[1].getClassName().split("\\.");
    String topologyName = className[className.length - 1];
    
    
    StormSubmitter.submitTopology(topologyName, conf, builder.createTopology());
}
 
Example 19
Source Project: jstorm   File: NettyClient.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings("rawtypes")
NettyClient(Map storm_conf, ChannelFactory factory, String host, int port, ReconnectRunnable reconnector) {
    this.stormConf = storm_conf;
    this.factory = factory;
    this.reconnector = reconnector;

    retries = new AtomicInteger(0);
    channelRef = new AtomicReference<>(null);
    beingClosed = new AtomicBoolean(false);
    pendings = new AtomicLong(0);

    nettyConnection = new NettyConnection();
    nettyConnection.setClientPort(NetWorkUtils.ip(), ConfigExtension.getLocalWorkerPort(storm_conf));
    nettyConnection.setServerPort(host, port);

    // Configure
    bufferSize = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_BUFFER_SIZE));
    maxRetries = Math.min(30, Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_RETRIES)));
    baseSleepMs = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MIN_SLEEP_MS));
    maxSleepMs = Utils.getInt(storm_conf.get(Config.STORM_MESSAGING_NETTY_MAX_SLEEP_MS));

    timeoutMs = ConfigExtension.getNettyPendingBufferTimeout(storm_conf);
    MAX_SEND_PENDING = (int) ConfigExtension.getNettyMaxSendPending(storm_conf);

    this.messageBatchSize = Utils.getInt(storm_conf.get(Config.STORM_NETTY_MESSAGE_BATCH_SIZE), 262144);
    this.messageBuffer = new MessageBuffer(messageBatchSize);

    this.writeLock = new Object();

    // Start the connection attempt.
    remoteAddr = new InetSocketAddress(host, port);
    name = remoteAddr.toString();
    connectMyself = isConnectMyself(stormConf, host, port);

    address = JStormServerUtils.getName(host, port);

    this.enableNettyMetrics = MetricUtils.isEnableNettyMetrics(storm_conf);
    NettyMetricInstance.register();
    LOG.info("* enable netty metrics: {}", this.enableNettyMetrics);
    if (!connectMyself) {
        registerMetrics();
    }
    closingChannel = new HashSet<Channel>();

    BATCH_THRESHOLD_WARN = ConfigExtension.getNettyBufferThresholdSize(stormConf);
}
 
Example 20
Source Project: jstorm   File: ShellBasedGroupsMapping.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Invoked once immediately after construction
 * 
 * @param storm_conf Storm configuration
 */
@Override
public void prepare(Map storm_conf) {
    int timeout = Utils.getInt(storm_conf.get(Config.STORM_GROUP_MAPPING_SERVICE_CACHE_DURATION_SECS));
    cachedGroups = new TimeCacheMap<String, Set<String>>(timeout);
}