Java Code Examples for backtype.storm.utils.Utils#sleep()

The following examples show how to use backtype.storm.utils.Utils#sleep() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: eagle   File: SampleClient1.java    License: Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) {
    long base = System.currentTimeMillis();
    AtomicLong msgCount = new AtomicLong();

    Config config = ConfigFactory.load();
    try (KafkaProducer<String, String> proceduer = createProceduer(config)) {
        while (true) {
            int hostIndex = 6;
            for (int i = 0; i < hostIndex; i++) {
                base = send_metric(base, proceduer, PERFMON_CPU_STREAM, i);
                msgCount.incrementAndGet();
                base = send_metric(base, proceduer, PERFMON_MEM_STREAM, i);
                msgCount.incrementAndGet();
            }

            if ((msgCount.get() % 600) == 0) {
                System.out.println("send 600 CPU/MEM metric!");
            }

            Utils.sleep(3000);
        }
    }
}
 
Example 2
@Override
public final void nextTuple() {
	final Status status = _queue.poll();
	if (null == status) {
		//If _queue is empty sleep the spout thread so it doesn't consume resources.
		Utils.sleep(500);
       } else {
		//Emit the complete tweet to the Bolt.
		this._outputCollector.emit(new Values(status));
	}
}
 
Example 3
@Override
public void run() {
  long now = System.currentTimeMillis();
  long endTime = now + config.totalTime;
  MetricsState state = new MetricsState();
  state.startTime = now;
  state.lastTime = now;

  final String path = config.path;
  final String name = config.name;
  final String confFile = String.format(
          MetricsCollectorConfig.CONF_FILE_FORMAT, path, name, now);
  final String dataFile = String.format(
          MetricsCollectorConfig.DATA_FILE_FORMAT, path, name, now);
  PrintWriter confWriter = FileUtils.createFileWriter(path, confFile);
  PrintWriter dataWriter = FileUtils.createFileWriter(path, dataFile);
  config.writeStormConfig(confWriter);
  writeHeader(dataWriter);

  try {
    boolean live = true;
    do {
      Utils.sleep(config.pollInterval);
      now = System.currentTimeMillis();
      live = pollNimbus(getNimbusClient(config.stormConfig), now, state, dataWriter);
    } while (live && now < endTime);
  } catch (Exception e) {
    LOG.error("storm metrics failed! ", e);
  } finally {
    dataWriter.close();
    confWriter.close();
  }
}
 
Example 4
public void nextTuple() {
  // We explicitly slow down the spout to avoid the stream mgr to be the bottleneck
  Utils.sleep(1);
  final String word = words[rand.nextInt(words.length)];
  // To enable acking, we need to emit tuple with MessageId, which is an object
  collector.emit(new Values(word), "MESSAGE_ID");
}
 
Example 5
public void nextTuple() {
  // We explicitly slow down the spout to avoid the stream mgr to be the bottleneck
  Utils.sleep(1);
  final String word = words[rand.nextInt(words.length)];
  // To enable acking, we need to emit tuple with MessageId, which is an object
  collector.emit(new Values(word), word);
}
 
Example 6
Source Project: storm-solr   File: SpringSpout.java    License: Apache License 2.0 5 votes vote down vote up
public void nextTuple() {

    if (maxPendingMessages > 0 && pendingMessages.size() >= maxPendingMessages) {
      if (log.isDebugEnabled()) {
        log.debug(String.format("Too many pending messages (%d), waiting until there are less than %d before emitting more.",
          pendingMessages.size(), maxPendingMessages));
      }

      Utils.sleep(50);
      return; // too many messages in-flight, don't provide anymore until we catch up
    }

    boolean hasNext = false;
    try {
      hasNext = dataProvider.next(reusableNamedValues);
    } catch (Exception exc) {
      log.error(String.format("Failed to get next record from %s due to: %s", spoutLogicBeanId, exc.toString()));
    }

    if (!hasNext) {
      Utils.sleep(50);
      return;
    }

    String messageId = reusableNamedValues.getMessageId();
    if (messageId == null) {
      messageId = String.valueOf(messageIdCounter.incrementAndGet());
    }
    _collector.emit(reusableNamedValues.values(), messageId);

    if (++emitted % 1000 == 0) {
      log.debug("emitted: " + emitted);
    }

    if (maxPendingMessages > 0) {
      pendingMessages.add(messageId); // keep track of this message as being "pending"
    }
  }
 
Example 7
public static void main(String[] args) {
  TopologyBuilder builder = new TopologyBuilder();

  builder.setSpout("commit-feed-listener", new CommitFeedListener());

  builder
      .setBolt("email-extractor", new EmailExtractor())
      .shuffleGrouping("commit-feed-listener");

  builder
      .setBolt("email-counter", new EmailCounter())
      .fieldsGrouping("email-extractor", new Fields("email"));

  Config config = new Config();
  config.setDebug(true);

  StormTopology topology = builder.createTopology();

  LocalCluster cluster = new LocalCluster();
  cluster.submitTopology("github-commit-count-topology",
      config,
      topology);

  Utils.sleep(TEN_MINUTES);
  cluster.killTopology("github-commit-count-topology");
  cluster.shutdown();
}
 
Example 8
Source Project: eagle   File: UnitTopologyRunner.java    License: Apache License 2.0 5 votes vote down vote up
private void run(String topologyId,
                 int numOfTotalWorkers,
                 int numOfSpoutTasks,
                 int numOfRouterBolts,
                 int numOfAlertBolts,
                 int numOfPublishExecutors,
                 int numOfPublishTasks,
                 Config config,
                 boolean localMode) {

    backtype.storm.Config stormConfig = givenStormConfig == null ? new backtype.storm.Config() : givenStormConfig;
    // TODO: Configurable metric consumer instance number

    int messageTimeoutSecs = config.hasPath(MESSAGE_TIMEOUT_SECS) ? config.getInt(MESSAGE_TIMEOUT_SECS) : DEFAULT_MESSAGE_TIMEOUT_SECS;
    LOG.info("Set topology.message.timeout.secs as {}", messageTimeoutSecs);
    stormConfig.setMessageTimeoutSecs(messageTimeoutSecs);

    if (config.hasPath("metric")) {
        stormConfig.registerMetricsConsumer(StormMetricTaggedConsumer.class, config.root().render(ConfigRenderOptions.concise()), 1);
    }

    stormConfig.setNumWorkers(numOfTotalWorkers);
    StormTopology topology = buildTopology(topologyId, numOfSpoutTasks, numOfRouterBolts, numOfAlertBolts, numOfPublishExecutors, numOfPublishTasks, config).createTopology();

    if (localMode) {
        LOG.info("Submitting as local mode");
        LocalCluster cluster = new LocalCluster();
        cluster.submitTopology(topologyId, stormConfig, topology);
        Utils.sleep(Long.MAX_VALUE);
    } else {
        LOG.info("Submitting as cluster mode");
        try {
            StormSubmitter.submitTopologyWithProgressBar(topologyId, stormConfig, topology);
        } catch (Exception ex) {
            LOG.error("fail submitting topology {}", topology, ex);
            throw new IllegalStateException(ex);
        }
    }
}
 
Example 9
@Override
public final void nextTuple() {
	final Status status = _queue.poll();
	if (null == status) {
		//If _queue is empty sleep the spout thread so it doesn't consume resources.
		Utils.sleep(500);
       } else {
		//Emit the complete tweet to the Bolt.
		this._outputCollector.emit(new Values(status));
	}
}
 
Example 10
@Override
public void nextTuple() {
    String message = fetchThread.pollMessage();
    if (message != null) {
        collector.emit(topic, new Values(message));
        _spoutMetric.incr();
    } else {
        Utils.sleep(100);
        warnningStep++;
        if (warnningStep % 100 == 0) {
            LOG.warn("Queue is empty, cannot poll message.");
        }
    }
}
 
Example 11
Source Project: eagle   File: SampleClient2.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @param args
 */
public static void main(String[] args) {
    AtomicLong base1 = new AtomicLong(System.currentTimeMillis());
    AtomicLong base2 = new AtomicLong(System.currentTimeMillis());
    AtomicLong count = new AtomicLong();

    Config config = ConfigFactory.load();

    try (KafkaProducer<String, String> proceduer = SampleClient1.createProceduer(config)) {
        while (true) {
            nextUuid = String.format(instanceUuidTemp, UUID.randomUUID().toString());
            nextReqId = String.format(reqIdTemp, UUID.randomUUID().toString());

            int hostIndex = 6;
            for (int i = 0; i < hostIndex; i++) {
                sendMetric(base1, base2, count, proceduer, i);
            }

            if (count.get() % 600 == 0) {
                System.out.println("send 600 LOG/FAILURE metric!");
            }

            Utils.sleep(3000);

        }
    }
}
 
Example 12
Source Project: jstorm   File: WordSpout.java    License: Apache License 2.0 5 votes vote down vote up
public void nextTuple() {
    Utils.sleep(100);
    final String[] words = new String[] { "nathan", "mike", "jackson", "golda", "bertels" };
    final Random rand = new Random();
    final String word = words[rand.nextInt(words.length)];
    _collector.emit(new Values(word));
}
 
Example 13
Source Project: eagle   File: ExampleStormApplication.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void nextTuple() {
    _collector.emit(Arrays.asList("disk.usage",System.currentTimeMillis(),"host_1",56.7));
    _collector.emit(Arrays.asList("cpu.usage",System.currentTimeMillis(),"host_2",99.8));
    Utils.sleep(100);
}
 
Example 14
Source Project: jstorm   File: RandomIntegerSpout.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void nextTuple() {
    Utils.sleep(100);
    collector.emit(new Values(rand.nextInt(1000), System.currentTimeMillis() - (24 * 60 * 60 * 1000), ++msgId),
            msgId);
}
 
Example 15
@Override
public void nextTuple() {
    Utils.sleep(10);
    collector.emit(new Values(rand.nextInt(1000), ip, port), msgId);
}
 
Example 16
Source Project: StormCV   File: MjpegStreamingOp.java    License: Apache License 2.0 4 votes vote down vote up
@GET @Path("/mjpeg/{streamid}.mjpeg")
@Produces("multipart/x-mixed-replace; boundary=--BoundaryString\r\n")
public Response mjpeg(@PathParam("streamid") final String streamId){
	StreamingOutput output = new StreamingOutput() {
		
		private BufferedImage prevImage = null;
		private int sleep = 1000/frameRate;
		
		@Override
		public void write(OutputStream outputStream) throws IOException, WebApplicationException {
			BufferedImage image = null;
			try{
				while((image = images.getIfPresent(streamId)) != null) /*synchronized(image)*/ {
					if(prevImage == null || !image.equals(prevImage)){
						ByteArrayOutputStream baos = new ByteArrayOutputStream();
						ImageIO.write(image, "jpg", baos);
						byte[] imageData = baos.toByteArray();
						 outputStream.write((
							        "--BoundaryString\r\n" +
							        "Content-type: image/jpeg\r\n" +
							        "Content-Length: "+imageData.length+"\r\n\r\n").getBytes());
						outputStream.write(imageData);
						outputStream.write("\r\n\r\n".getBytes());
						outputStream.flush();
					}
					Utils.sleep(sleep);
					/*
					try {
						image.notifyAll();
						image.wait();
					} catch (InterruptedException e) {
						// just read the next image
					}
					*/
				}
				outputStream.flush();
				outputStream.close();
			}catch(IOException ioe){
				logger.info("Steam for ["+streamId+"] closed by client!");
			}
		}
	};
	return Response.ok(output)
			.header("Connection", "close")
			.header("Max-Age", "0")
			.header("Expires", "0")
			.header("Cache-Control", "no-cache, private")
			.header("Pragma", "no-cache")
			.build();
}
 
Example 17
public static void main(String[] args){
	// first some global (topology configuration)
	StormCVConfig conf = new StormCVConfig();

	/**
	 * Sets the OpenCV library to be used which depends on the system the topology is being executed on
	 */
	conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib");
	//conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "win64_opencv_java248.dll");
	
	conf.setNumWorkers(3); // number of workers in the topology
	conf.setMaxSpoutPending(32); // maximum un-acked/un-failed frames per spout (spout blocks if this number is reached)
	conf.put(StormCVConfig.STORMCV_FRAME_ENCODING, Frame.JPG_IMAGE); // indicates frames will be encoded as JPG throughout the topology (JPG is the default when not explicitly set)
	conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true); // True if Storm should timeout messages or not.
	conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS , 10); // The maximum amount of time given to the topology to fully process a message emitted by a spout (default = 30)
	conf.put(StormCVConfig.STORMCV_SPOUT_FAULTTOLERANT, false); // indicates if the spout must be fault tolerant; i.e. spouts do NOT! replay tuples on fail
	conf.put(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC, 30); // TTL (seconds) for all elements in all caches throughout the topology (avoids memory overload)

	String userDir = System.getProperty("user.dir").replaceAll("\\\\", "/");
	
	// create a list with files to be processed, in this case just one. Multiple files will be spread over the available spouts
	List<String> files = new ArrayList<String>();
	files.add( "file://"+ userDir +"/resources/data/" ); // actual use of this directory depends on the fetcher. The ImageFetcher will read all image files present within the directory.

	// now create the topology itself (spout -> facedetection --> drawer)
	TopologyBuilder builder = new TopologyBuilder();
	 // just one spout reading images from a directory, sleeping 100ms after each file was read
	builder.setSpout("spout", new CVParticleSpout( new ImageFetcher(files).sleepTime(100) ), 1 );
	
	// one bolt with a HaarCascade classifier with the lbpcascade_frontalface model to detecting faces. This operation outputs a Frame including the Features with detected faces
	builder.setBolt("face_detect", new SingleInputBolt(
		new HaarCascadeOp("face", "lbpcascade_frontalface.xml")
				.outputFrame(true)
		), 1)
		.shuffleGrouping("spout");
			
	// The bounding boxes of the Face Feature are extracted from the Frame and emitted as separate frames
	builder.setBolt("face_extraction", new SingleInputBolt(
			new ROIExtractionOp("face").spacing(25)
		), 1)
		.shuffleGrouping("face_detect");
			
	// simple bolt that draws Features (i.e. locations of features) into the frame and writes the frame to the local file system at /output/facedetections
	builder.setBolt("drawer", new SingleInputBolt(new DrawFeaturesOp().destination("file://"+userDir+ "/output/facedetections/")), 1)
		.shuffleGrouping("face_extraction");
	
	try {
		// run in local mode
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology( "facedetection", conf, builder.createTopology() );
		Utils.sleep(300*1000); // run for some time and then kill the topology
		cluster.shutdown();
		System.exit(1);
		
		// run on a storm cluster
		// StormSubmitter.submitTopology("some_topology_name", conf, builder.createTopology());
	} catch (Exception e){
		e.printStackTrace();
	}
}
 
Example 18
public static void main(String[] args){
	// first some global (topology configuration)
	StormCVConfig conf = new StormCVConfig();
	
	/**
	 * Sets the OpenCV library to be used which depends on the system the topology is being executed on
	 */
	conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib");
	
	conf.setNumWorkers(3); // number of workers in the topology
	conf.setMaxSpoutPending(32); // maximum un-acked/un-failed frames per spout (spout blocks if this number is reached)
	conf.put(StormCVConfig.STORMCV_FRAME_ENCODING, Frame.JPG_IMAGE); // indicates frames will be encoded as JPG throughout the topology (JPG is the default when not explicitly set)
	conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true); // True if Storm should timeout messages or not.
	conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS , 10); // The maximum amount of time given to the topology to fully process a message emitted by a spout (default = 30)
	conf.put(StormCVConfig.STORMCV_SPOUT_FAULTTOLERANT, false); // indicates if the spout must be fault tolerant; i.e. spouts do NOT! replay tuples on fail
	conf.put(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC, 30); // TTL (seconds) for all elements in all caches throughout the topology (avoids memory overload)
	conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib"); // sets the opencv lib to be used by all OpenCVOperation implementing operations
	
	List<String> urls = new ArrayList<String>();
	urls.add( "rtsp://streaming3.webcam.nl:1935/n224/n224.stream" );
	urls.add("rtsp://streaming3.webcam.nl:1935/n233/n233.stream");

	@SuppressWarnings("rawtypes")
	List<ISingleInputOperation> operations = new ArrayList<ISingleInputOperation>();
	operations.add(new FeatureExtractionOp("sift", FeatureDetector.SIFT, DescriptorExtractor.SIFT).outputFrame(true));
	operations.add(new DrawFeaturesOp() );
	 
	int frameSkip = 13; 
	
	TopologyBuilder builder = new TopologyBuilder();
	builder.setSpout("spout", new CVParticleSpout( 
			new FetchAndOperateFetcher( // use a meta fetcher to combine a Fetcher and Operation
					new StreamFrameFetcher(urls).frameSkip(frameSkip), // use a normal fetcher to get video frames from streams
					new SequentialFrameOp(operations).outputFrame(true).retainImage(true) // use a sequential operation to execute a number of operations on frames
				)
			) , 2 );
	
	// add bolt that creates a webservice on port 8558 enabling users to view the result
	builder.setBolt("streamer", new BatchInputBolt(
			new SlidingWindowBatcher(2, frameSkip).maxSize(6), // note the required batcher used as a buffer and maintains the order of the frames
			new MjpegStreamingOp().port(8558).framerate(5)).groupBy(new Fields(FrameSerializer.STREAMID))
		, 1)
		.shuffleGrouping("spout");
	
	try {
		
		// run in local mode
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology( "sequential_spout", conf, builder.createTopology() );
		Utils.sleep(120*1000); // run two minutes and then kill the topology
		cluster.shutdown();
		System.exit(1);
		
		// run on a storm cluster
		// StormSubmitter.submitTopology("some_topology_name", conf, builder.createTopology());
	} catch (Exception e){
		e.printStackTrace();
	}
}
 
Example 19
public static void main(String[] args){
	// first some global (topology configuration)
	StormCVConfig conf = new StormCVConfig();

	/**
	 * Sets the OpenCV library to be used which depends on the system the topology is being executed on
	 */
	conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib");
	
	conf.setNumWorkers(8); // number of workers in the topology
	conf.setMaxSpoutPending(32); // maximum un-acked/un-failed frames per spout (spout blocks if this number is reached)
	conf.put(StormCVConfig.STORMCV_FRAME_ENCODING, Frame.JPG_IMAGE); // indicates frames will be encoded as JPG throughout the topology (JPG is the default when not explicitly set)
	conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true); // True if Storm should timeout messages or not.
	conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS , 10); // The maximum amount of time given to the topology to fully process a message emitted by a spout (default = 30)
	conf.put(StormCVConfig.STORMCV_SPOUT_FAULTTOLERANT, false); // indicates if the spout must be fault tolerant; i.e. spouts do NOT! replay tuples on fail
	conf.put(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC, 30); // TTL (seconds) for all elements in all caches throughout the topology (avoids memory overload)
	
	String userDir = System.getProperty("user.dir").replaceAll("\\\\", "/");
	// create a list with files to be processed, in this case just one. Multiple files will be spread over the available spouts
	List<String> files = new ArrayList<String>();
	files.add( "file://"+ userDir + "/resources/data/" );

	int frameSkip = 13; 
	
	// now create the topology itself (spout -> scale -> {face detection, sift} -> drawer -> streamer)
	TopologyBuilder builder = new TopologyBuilder();
	 // just one spout reading video files, extracting 1 frame out of 25 (i.e. 1 per second)
	builder.setSpout("spout", new CVParticleSpout( new FileFrameFetcher(files).frameSkip(frameSkip) ), 1 );
	
	// add bolt that scales frames down to 25% of the original size 
	builder.setBolt("scale", new SingleInputBolt( new ScaleImageOp(0.25f)), 1)
		.shuffleGrouping("spout");
	
	// one bolt with a HaarCascade classifier detecting faces. This operation outputs a Frame including the Features with detected faces.
	// the xml file must be present on the classpath!
	builder.setBolt("face", new SingleInputBolt( new HaarCascadeOp("face", "lbpcascade_frontalface.xml").outputFrame(true)), 1)
		.shuffleGrouping("scale");
	
	// add a bolt that performs SIFT keypoint extraction
	builder.setBolt("sift", new SingleInputBolt( new FeatureExtractionOp("sift", FeatureDetector.SIFT, DescriptorExtractor.SIFT).outputFrame(false)), 2)
		.shuffleGrouping("scale");
	
	// Batch bolt that waits for input from both the face and sift detection bolts and combines them in a single frame object
	builder.setBolt("combiner", new BatchInputBolt(new SequenceNrBatcher(2), new FeatureCombinerOp()), 1)
		.fieldsGrouping("sift", new Fields(FrameSerializer.STREAMID))
		.fieldsGrouping("face", new Fields(FrameSerializer.STREAMID));
	
	// simple bolt that draws Features (i.e. locations of features) into the frame
	builder.setBolt("drawer", new SingleInputBolt(new DrawFeaturesOp()), 1)
		.shuffleGrouping("combiner");
	
	// add bolt that creates a webservice on port 8558 enabling users to view the result
	builder.setBolt("streamer", new BatchInputBolt(
			new SlidingWindowBatcher(2, frameSkip).maxSize(6), // note the required batcher used as a buffer and maintains the order of the frames
			new MjpegStreamingOp().port(8558).framerate(5)).groupBy(new Fields(FrameSerializer.STREAMID))
		, 1)
		.shuffleGrouping("drawer");

	// NOTE: if the topology is started (locally) go to http://localhost:8558/streaming/tiles and click the image to see the stream!
	
	try {
		
		// run in local mode
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology( "multifeature", conf, builder.createTopology() );
		Utils.sleep(120*1000); // run two minutes and then kill the topology
		cluster.shutdown();
		System.exit(1);
		
		// run on a storm cluster
		// StormSubmitter.submitTopology("some_topology_name", conf, builder.createTopology());
	} catch (Exception e){
		e.printStackTrace();
	}
}
 
Example 20
public static void main(String[] args){
	// first some global (topology configuration)
	StormCVConfig conf = new StormCVConfig();

	/**
	 * Sets the OpenCV library to be used which depends on the system the topology is being executed on
	 */
	conf.put(StormCVConfig.STORMCV_OPENCV_LIB, "mac64_opencv_java248.dylib");

	conf.setNumWorkers(6); // number of workers in the topology
	conf.setMaxSpoutPending(32); // maximum un-acked/un-failed frames per spout (spout blocks if this number is reached)
	conf.put(StormCVConfig.STORMCV_FRAME_ENCODING, Frame.JPG_IMAGE); // indicates frames will be encoded as JPG throughout the topology (JPG is the default when not explicitly set)
	conf.put(Config.TOPOLOGY_ENABLE_MESSAGE_TIMEOUTS, true); // True if Storm should timeout messages or not.
	conf.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS , 10); // The maximum amount of time given to the topology to fully process a message emitted by a spout (default = 30)
	conf.put(StormCVConfig.STORMCV_SPOUT_FAULTTOLERANT, false); // indicates if the spout must be fault tolerant; i.e. spouts do NOT! replay tuples on fail
	conf.put(StormCVConfig.STORMCV_CACHES_TIMEOUT_SEC, 30); // TTL (seconds) for all elements in all caches throughout the topology (avoids memory overload)
	
	String userDir = System.getProperty("user.dir").replaceAll("\\\\", "/");
	List<String> urls = new ArrayList<String>();
	urls.add( "file://"+ userDir +"/resources/data/The_Nut_Job_trailer.mp4" );

	int frameSkip = 13;
	
	TopologyBuilder builder = new TopologyBuilder();
	
	// spout reading streams emitting 2 frames group of frames (containing 2 frames) to the optical flow operation
	//builder.setSpout("spout", new CVParticleSpout( new StreamFrameFetcher(urls).frameSkip(frameSkip).groupSize(2) ), 1 ).setNumTasks(1);
	builder.setSpout("spout", new CVParticleSpout( new FileFrameFetcher(urls).frameSkip(frameSkip).groupSize(2) ), 1 ).setNumTasks(1);
	
	// add bolt that scales frames down to 50% of the original size 
	builder.setBolt("scale", new SingleInputBolt( new ScaleImageOp(0.5f)), 1)
		.shuffleGrouping("spout");
	
	builder.setBolt("grouper", new BatchInputBolt(new DiscreteWindowBatcher(2, 1), new FrameGrouperOp()), 1)
		.fieldsGrouping("scale",new Fields(FrameSerializer.STREAMID));
	
	// two group of frames operation calculating Optical Flow on Groups it receives. This makes the optical flow stateless! 
	builder.setBolt("optical_flow", new SingleInputBolt(
				new GroupOfFramesOp(new OpticalFlowOp("optical flow"))
			), 2).shuffleGrouping("grouper");
	
	// simple operation that visualizes the optical flow and puts it into a Frame object that can be shown / streamed
	builder.setBolt("flow_viz", new SingleInputBolt(new OpticalFlowVisualizeOp("optical flow")), 1
			).shuffleGrouping("optical_flow");
	
	// add bolt that creates a webservice on port 8558 enabling users to view the result
	builder.setBolt("streamer", new BatchInputBolt(
			new SlidingWindowBatcher(2, frameSkip).maxSize(6), 
			new MjpegStreamingOp().port(8558).framerate(5)).groupBy(new Fields(FrameSerializer.STREAMID))
		, 1)
		.shuffleGrouping("flow_viz");
	
	// NOTE: if the topology is started (locally) go to http://localhost:8558/streaming/tiles and click the image to see the stream!
	
	try {
		
		// run in local mode
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology( "bigflow", conf, builder.createTopology() );
		Utils.sleep(120*1000); // run for one minute and then kill the topology
		cluster.shutdown();
		System.exit(1);
		
		// run on a storm cluster
		// StormSubmitter.submitTopology("some_topology_name", conf, builder.createTopology());
	} catch (Exception e){
		e.printStackTrace();
	}
}