backtype.storm.task.IMetricsContext Java Examples
The following examples show how to use
backtype.storm.task.IMetricsContext.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HBaseMapState.java From storm-hbase with Apache License 2.0 | 6 votes |
@SuppressWarnings({"rawtypes", "unchecked"}) public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("Preparing HBase State for partition {} of {}.", partitionIndex + 1, numPartitions); IBackingMap state = new HBaseMapState(options, conf, partitionIndex); if(options.cacheSize > 0) { state = new CachedMap(state, options.cacheSize); } MapState mapState; switch (stateType) { case NON_TRANSACTIONAL: mapState = NonTransactionalMap.build(state); break; case OPAQUE: mapState = OpaqueMap.build(state); break; case TRANSACTIONAL: mapState = TransactionalMap.build(state); break; default: throw new IllegalArgumentException("Unknown state type: " + stateType); } return new SnapshottableMap(mapState, new Values(options.globalKey)); }
Example #2
Source File: ESIndexMapState.java From storm-trident-elasticsearch with Apache License 2.0 | 5 votes |
@Override public State makeState(Map conf, IMetricsContext iMetricsContext, int i, int i2) { Options options = new Options(conf); ESIndexMapState<TransactionalValue<T>> mapState = new ESIndexMapState<>(clientFactory.makeClient(conf), serializer, new BulkResponseHandler.LoggerResponseHandler(), options.reportError()); MapState<T> ms = TransactionalMap.build(new CachedMap(mapState, options.getCachedMapSize())); Values snapshotKey = new Values(options.getGlobalKey()); return new SnapshottableMap<>(ms, snapshotKey); }
Example #3
Source File: ElasticSearchStateFactory.java From trident-tutorial with Apache License 2.0 | 5 votes |
@Override public State makeState(Map map, IMetricsContext iMetricsContext, int partitionIndex, int numPartitions) { /** * Here, we're using a singleton because we're connecting to a local in-memory ES-cluster. * In a real world application, you should just instantiate a client to connect * to your production database. */ Client client = ElasticSearchSingleton.getInstance(); return new ElasticSearchState(client); }
Example #4
Source File: HdfsStateFactory.java From storm-hdfs with Apache License 2.0 | 5 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); return state; }
Example #5
Source File: CassandraCqlStateFactory.java From storm-cassandra-cql with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public State makeState(Map configuration, IMetricsContext metrics, int partitionIndex, int numPartitions) { // worth synchronizing here? if (clientFactory == null) { clientFactory = new MapConfiguredCqlClientFactory(configuration); } final String maxBatchSizeString = (String) configuration.get(CassandraCqlStateFactory.TRIDENT_CASSANDRA_MAX_BATCH_SIZE); final int maxBatchSize = (maxBatchSizeString == null) ? DEFAULT_MAX_BATCH_SIZE : Integer.parseInt((String) maxBatchSizeString); LOG.debug("Creating State for partition [{}] of [{}]", new Object[]{partitionIndex, numPartitions}); return new CassandraCqlState(clientFactory, maxBatchSize, batchConsistencyLevel); }
Example #6
Source File: CassandraCqlIncrementalStateFactory.java From storm-cassandra-cql with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public State makeState(Map configuration, IMetricsContext metrics, int partitionIndex, int numPartitions) { // NOTE: Lazy instantiation because Cluster is not serializable. if (clientFactory == null) { clientFactory = new MapConfiguredCqlClientFactory(configuration); } LOG.debug("Creating State for partition [{}] of [{}]", new Object[]{partitionIndex, numPartitions}); return new CassandraCqlIncrementalState<K, V>(clientFactory, aggregator, mapper, partitionIndex); }
Example #7
Source File: CassandraCqlMapState.java From storm-cassandra-cql with Apache License 2.0 | 5 votes |
@SuppressWarnings("rawtypes") public void registerMetrics(Map conf, IMetricsContext context, String mapStateMetricName) { int bucketSize = (Integer) (conf.get(Config.TOPOLOGY_BUILTIN_METRICS_BUCKET_SIZE_SECS)); String metricBaseName = "cassandra/" + mapStateMetricName; _mreads = context.registerMetric(metricBaseName + "/readCount", new CountMetric(), bucketSize); _mwrites = context.registerMetric(metricBaseName + "/writeCount", new CountMetric(), bucketSize); _mexceptions = context.registerMetric(metricBaseName + "/exceptionCount", new CountMetric(), bucketSize); }
Example #8
Source File: CassandraCqlMapStateFactory.java From storm-cassandra-cql with Apache License 2.0 | 5 votes |
@SuppressWarnings({"rawtypes", "unchecked"}) public State makeState(Map configuration, IMetricsContext metrics, int partitionIndex, int numPartitions) { if (clientFactory == null) { clientFactory = new MapConfiguredCqlClientFactory(configuration); } Session session; if(options.keyspace != null) { session = clientFactory.getSession(options.keyspace); } else { session = clientFactory.getSession(); } CassandraCqlMapState state = new CassandraCqlMapState(session, mapper, options, configuration); state.registerMetrics(configuration, metrics, options.mapStateMetricName); CachedMap cachedMap = new CachedMap(state, options.localCacheSize); MapState mapState; if (stateType == StateType.NON_TRANSACTIONAL) { mapState = NonTransactionalMap.build(cachedMap); } else if (stateType == StateType.OPAQUE) { mapState = OpaqueMap.build(cachedMap); } else if (stateType == StateType.TRANSACTIONAL) { mapState = TransactionalMap.build(cachedMap); } else { throw new RuntimeException("Unknown state type: " + stateType); } return new SnapshottableMap(mapState, new Values(options.globalKey)); }
Example #9
Source File: HdfsStateFactory.java From jstorm with Apache License 2.0 | 5 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { LOG.info("makeState(partitonIndex={}, numpartitions={}", partitionIndex, numPartitions); HdfsState state = new HdfsState(this.options); state.prepare(conf, metrics, partitionIndex, numPartitions); return state; }
Example #10
Source File: ESIndexMapState.java From storm-trident-elasticsearch with Apache License 2.0 | 5 votes |
@Override public State makeState(Map conf, IMetricsContext iMetricsContext, int i, int i2) { Options options = new Options(conf); ESIndexMapState<OpaqueValue<T>> mapState = new ESIndexMapState<>(clientFactory.makeClient(conf), serializer, new BulkResponseHandler.LoggerResponseHandler(), options.reportError()); MapState ms = OpaqueMap.build(new CachedMap(mapState, options.getCachedMapSize())); return new SnapshottableMap<OpaqueValue<T>>(ms, new Values(options.getGlobalKey())); }
Example #11
Source File: ESIndexMapState.java From storm-trident-elasticsearch with Apache License 2.0 | 5 votes |
@Override public State makeState(Map conf, IMetricsContext iMetricsContext, int i, int i2) { Options options = new Options(conf); ESIndexMapState<T> mapState = new ESIndexMapState<>(clientFactory.makeClient(conf), serializer, new BulkResponseHandler.LoggerResponseHandler(), options.reportError()); MapState<T> ms = NonTransactionalMap.build(new CachedMap<>(mapState, options.getCachedMapSize())); return new SnapshottableMap<>(ms, new Values(options.getGlobalKey())); }
Example #12
Source File: HdfsState.java From jstorm with Apache License 2.0 | 4 votes |
void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { this.options.prepare(conf, partitionIndex, numPartitions); initLastTxn(conf, partitionIndex); }
Example #13
Source File: WindowsStateFactory.java From jstorm with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new WindowsState(); }
Example #14
Source File: LRUMemoryMapState.java From jstorm with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new LRUMemoryMapState(_maxSize, _id + partitionIndex); }
Example #15
Source File: MemoryMapState.java From jstorm with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new MemoryMapState(_id + partitionIndex); }
Example #16
Source File: TridentReach.java From jstorm with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new StaticSingleKeyMapState(_map); }
Example #17
Source File: FirstStoryDetection.java From first-stories-twitter with MIT License | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new RecentTweetsDB(Integer.valueOf((String) conf .get("RECENT_TWEETS_TO_COMPARE_WITH")), numPartitions); }
Example #18
Source File: IndexStateFactory.java From trident-elasticsearch with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext iMetricsContext, int i, int i2) { Client client = this.clientFactory.makeClient(conf); return new IndexState(client, this.exceptionHandler, this.batchSize); }
Example #19
Source File: FirstStoryDetection.java From first-stories-twitter with MIT License | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new BucketsDB(partialL, k, queueSize); }
Example #20
Source File: Part05_AdvancedStateAndDRPC.java From trident-tutorial with Apache License 2.0 | 4 votes |
private static StormTopology externalState(LocalDRPC drpc, FeederBatchSpout spout) { TridentTopology topology = new TridentTopology(); // You can reference existing data sources as well. // Here we are mocking up a "database" StateFactory stateFactory = new StateFactory() { @Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { MemoryMapState<Integer> name_to_age = new MemoryMapState<Integer>("name_to_age"); // This is a bit hard to read but it's just pre-populating the state List<List<Object>> keys = getKeys("ted", "mary", "jason", "tom", "chuck"); name_to_age.multiPut(keys, ImmutableList.of(32, 21, 45, 52, 18)); return name_to_age; } }; TridentState nameToAge = topology.newStaticState(stateFactory); // Let's setup another state that keeps track of actor's appearance counts per location TridentState countState = topology .newStream("spout", spout) .groupBy(new Fields("actor","location")) .persistentAggregate(new MemoryMapState.Factory(), new Count(), new Fields("count")); // Now, let's calculate the average age of actors seen topology .newDRPCStream("age_stats", drpc) .stateQuery(countState, new TupleCollectionGet(), new Fields("actor", "location")) .stateQuery(nameToAge, new Fields("actor"), new MapGet(), new Fields("age")) .each(new Fields("actor","location","age"), new Print()) .groupBy(new Fields("location")) .chainedAgg() .aggregate(new Count(), new Fields("count")) .aggregate(new Fields("age"), new Sum(), new Fields("sum")) .chainEnd() .each(new Fields("sum", "count"), new DivideAsDouble(), new Fields("avg")) .project(new Fields("location", "count", "avg")) ; return topology.build(); }
Example #21
Source File: HazelCastStateFactory.java From trident-tutorial with Apache License 2.0 | 4 votes |
@Override public MapState<T> makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { IBackingMap im = new HazelCastState<T>(new HazelCastHandler<TransactionalValue<T>>()); return TransactionalMap.build(im); }
Example #22
Source File: TridentReach.java From flink-perf with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new StaticSingleKeyMapState(_map); }
Example #23
Source File: HdfsState.java From storm-hdfs with Apache License 2.0 | 4 votes |
void prepare(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions){ this.options.prepare(conf, partitionIndex, numPartitions); }
Example #24
Source File: HBaseStateFactory.java From storm-hbase with Apache License 2.0 | 4 votes |
@Override public State makeState(Map map, IMetricsContext iMetricsContext, int partitionIndex, int numPartitions) { HBaseState state = new HBaseState(map , partitionIndex, numPartitions, options); state.prepare(); return state; }
Example #25
Source File: DRPC.java From storm-benchmark with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new StaticSingleKeyMapState(map); }
Example #26
Source File: DruidStateFactory.java From storm-example with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { DruidStateFactory.startRealtime(); return new DruidState(partitionIndex); }
Example #27
Source File: OutbreakTrendFactory.java From storm-example with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { return new OutbreakTrendState(new OutbreakTrendBackingMap()); }
Example #28
Source File: DruidStateFactory.java From storm-example with Apache License 2.0 | 4 votes |
@Override public State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions) { DruidStateFactory.startRealtime(); return new DruidState(); }
Example #29
Source File: ESIndexState.java From storm-trident-elasticsearch with Apache License 2.0 | 4 votes |
@Override public State makeState(Map map, IMetricsContext iMetricsContext, int i, int i2) { return new ESIndexState<>(makeClient(map), serializer); }
Example #30
Source File: StateFactory.java From jstorm with Apache License 2.0 | votes |
State makeState(Map conf, IMetricsContext metrics, int partitionIndex, int numPartitions);