org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy Java Examples

The following examples show how to use org.apache.storm.hdfs.bolt.rotation.TimedRotationPolicy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SourceHandler.java    From metron with Apache License 2.0 6 votes vote down vote up 
private void initialize() throws IOException {
  LOG.debug("Initializing Source Handler");
  this.fs = FileSystem.get(new Configuration());
  this.currentFile = createOutputFile();
  LOG.debug("Source Handler initialized with starting file: {}", currentFile);
  if(this.rotationPolicy instanceof TimedRotationPolicy){
    long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
    this.rotationTimer = new Timer(true);
    TimerTask task = new TimerTask() {
      @Override
      public void run() {
        try {
          LOG.debug("Rotating output file from TimerTask");
          rotateOutputFile();
        } catch(IOException e){
          LOG.warn("IOException during scheduled file rotation.", e);
        }
      }
    };
    this.rotationTimer.scheduleAtFixedRate(task, interval, interval);
  }
}
Example #2
Source File: HdfsFileTopology.java    From storm-hdfs with Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);

    SentenceSpout spout = new SentenceSpout();

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/foo/")
            .withExtension(".txt");



    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat()
            .withFieldDelimiter("|");

    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);

    HdfsBolt bolt = new HdfsBolt()
            .withConfigKey("hdfs.config")
            .withFsUrl(args[0])
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4)
            .shuffleGrouping(SENTENCE_SPOUT_ID);

    if (args.length == 2) {
        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(120);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[0], config, builder.createTopology());
    } else{
        System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>");
    }
}