Java Code Examples for org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy

The following are top voted examples for showing how to use org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: hadooparchitecturebook   File: MovingAvgLocalTopologyRunner.java   Source Code and License 6 votes vote down vote up
/**
 * Create bolt which will persist ticks to HDFS.
 */
private static HdfsBolt createHdfsBolt() {

  // Use "|" instead of "," for field delimiter:
  RecordFormat format = new DelimitedRecordFormat()
    .withFieldDelimiter("|");
  // sync the filesystem after every 1k tuples:
  SyncPolicy syncPolicy = new CountSyncPolicy(100);

  // Rotate files when they reach 5MB:
  FileRotationPolicy rotationPolicy = 
    new FileSizeRotationPolicy(5.0f, Units.MB);

  // Write records to <user>/stock-ticks/ directory in HDFS:
  FileNameFormat fileNameFormat = new DefaultFileNameFormat()
    .withPath("stock-ticks/");

  HdfsBolt hdfsBolt = new HdfsBolt()
    .withFsUrl("hdfs://localhost:8020")
    .withFileNameFormat(fileNameFormat)
    .withRecordFormat(format)
    .withRotationPolicy(rotationPolicy)
    .withSyncPolicy(syncPolicy);

  return hdfsBolt;
}
 
Example 2
Project: fksm   File: KafkaTopology.java   Source Code and License 6 votes vote down vote up
private static HdfsBolt buildHdfsBolt(String hdfsUrl,String prefix, Fields fields){
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat()
            .withFieldDelimiter(" : ").withFields(fields);

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/storm/").withPrefix(prefix).withExtension(".seq");

    HdfsBolt hdfsBolt = new HdfsBolt()
            .withFsUrl(hdfsUrl)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy)
            .withRetryCount(5)
            .addRotationAction(new MoveStormToLogAction().withDestination("/log"));

    return hdfsBolt;
}
 
Example 3
Project: erad2016-streamprocessing   File: SentimentAnalysisTopology.java   Source Code and License 6 votes vote down vote up
private static HdfsBolt createHdfsBolt() {
    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat()
            .withFieldDelimiter("|");

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath(Properties.getString("sa.storm.hdfs_output_file"));

    return new HdfsBolt()
            .withFsUrl(Properties.getString("sa.storm.hdfs_url"))
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
}
 
Example 4
Project: storm-kafka-hdfs-example   File: HdfsBoltConfigBuilder.java   Source Code and License 6 votes vote down vote up
public static FileRotationPolicy getTimeBasedFileRotationPolicy(String rotationUnits, int rotationCount) {
    TimedRotationPolicy.TimeUnit units;
    if (rotationUnits.toUpperCase().equals("SECONDS")) {
        units = TimedRotationPolicy.TimeUnit.SECONDS;
    } else if (rotationUnits.toUpperCase().equals("MINUTES")) {
        units = TimedRotationPolicy.TimeUnit.MINUTES;
    } else if (rotationUnits.toUpperCase().equals("HOURS")) {
        units = TimedRotationPolicy.TimeUnit.HOURS;
    } else if (rotationUnits.toUpperCase().equals("DAYS")) {
        units = TimedRotationPolicy.TimeUnit.DAYS;
    } else {
        units = TimedRotationPolicy.TimeUnit.MINUTES;
    }

    return new TimedRotationPolicy(rotationCount, units);
}
 
Example 5
Project: storm-kafka-hdfs-example   File: KafkaHdfsTopologyTest.java   Source Code and License 6 votes vote down vote up
/**
 * Returns the HdfsBoltConfigBuilder ready for use
 * @return HdfsBoltConfigBuilder Used to construct the kafka spout configuration
 */
private HdfsBoltConfigBuilder getHdfsBoltConfigBuilder() {
    FileRotationPolicy fileRotationPolicy = HdfsBoltConfigBuilder.getTimeBasedFileRotationPolicy(
            propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_ROTATION_POLICY_UNITS_KEY),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_ROTATION_POLICY_COUNT_KEY)));

    return new HdfsBoltConfigBuilder.Builder()
            .setFieldDelimiter(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_FIELD_DELIMITER_KEY))
            .setOutputLocation(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_OUTPUT_LOCATION_KEY))
            .setHdfsDefaultFs(propertyParser.getProperty(ConfigVars.HDFS_DEFAULT_FS_KEY))
            .setSyncCount(Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_SYNC_COUNT_KEY)))
            .setBoltParallelism(Integer.parseInt(
                    propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_PARALLELISM_KEY)))
            .setBoltName(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_NAME_KEY))
            .setFileRotationPolicy(fileRotationPolicy)
            .build();
}
 
Example 6
Project: storm-topology-examples   File: ConfigureHdfsBolt.java   Source Code and License 6 votes vote down vote up
private static FileRotationPolicy getSizeBasedFileRotationPolicy(String unitsConfigured, int sizeConfigured) {
    FileSizeRotationPolicy.Units units;
    if (unitsConfigured.toUpperCase().equals("KB")) {
        units = FileSizeRotationPolicy.Units.KB;
    } else if (unitsConfigured.toUpperCase().equals("MB")) {
        units = FileSizeRotationPolicy.Units.MB;
    } else if (unitsConfigured.toUpperCase().equals("GB")) {
        units = FileSizeRotationPolicy.Units.GB;
    } else if (unitsConfigured.toUpperCase().equals("TB")) {
        units = FileSizeRotationPolicy.Units.TB;
    } else {
        units = FileSizeRotationPolicy.Units.MB;
    }

    return new FileSizeRotationPolicy(sizeConfigured, units);
}
 
Example 7
Project: storm-topology-examples   File: ConfigureHdfsBolt.java   Source Code and License 6 votes vote down vote up
private static FileRotationPolicy getTimeBasedFileRotationPolicy(String unitsConfigured, int timeConfigured) {
    TimedRotationPolicy.TimeUnit units;
    if (unitsConfigured.toUpperCase().equals("SECONDS")) {
        units = TimedRotationPolicy.TimeUnit.SECONDS;
    } else if (unitsConfigured.toUpperCase().equals("MINUTES")) {
        units = TimedRotationPolicy.TimeUnit.MINUTES;
    } else if (unitsConfigured.toUpperCase().equals("HOURS")) {
        units = TimedRotationPolicy.TimeUnit.HOURS;
    } else if (unitsConfigured.toUpperCase().equals("DAYS")) {
        units = TimedRotationPolicy.TimeUnit.DAYS;
    } else {
        units = TimedRotationPolicy.TimeUnit.MINUTES;
    }

    return new TimedRotationPolicy(timeConfigured, units);
}
 
Example 8
Project: storm-smoke-test   File: ConnectorUtil.java   Source Code and License 6 votes vote down vote up
public static HdfsBolt getHdfsBolt(String fsUrl, String srcDir, String rotationDir) {
    // sync the filesystem after every tuple
    SyncPolicy syncPolicy = new CountSyncPolicy(1);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath(srcDir)
            .withExtension(".txt");

    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(",");
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1f, FileSizeRotationPolicy.Units.KB);

    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(fsUrl)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withSyncPolicy(syncPolicy)
            .withRotationPolicy(rotationPolicy)
            .addRotationAction(new MoveFileAction().toDestination(rotationDir));

    return bolt;
}
 
Example 9
Project: hadoop-arch-book   File: MovingAvgLocalTopologyRunner.java   Source Code and License 6 votes vote down vote up
/**
 * Create bolt which will persist ticks to HDFS.
 */
private static HdfsBolt createHdfsBolt() {

  // Use "|" instead of "," for field delimiter:
  RecordFormat format = new DelimitedRecordFormat()
    .withFieldDelimiter("|");
  // sync the filesystem after every 1k tuples:
  SyncPolicy syncPolicy = new CountSyncPolicy(100);

  // Rotate files when they reach 5MB:
  FileRotationPolicy rotationPolicy = 
    new FileSizeRotationPolicy(5.0f, Units.MB);

  // Write records to <user>/stock-ticks/ directory in HDFS:
  FileNameFormat fileNameFormat = new DefaultFileNameFormat()
    .withPath("stock-ticks/");

  HdfsBolt hdfsBolt = new HdfsBolt()
    .withFsUrl("hdfs://localhost:8020")
    .withFileNameFormat(fileNameFormat)
    .withRecordFormat(format)
    .withRotationPolicy(rotationPolicy)
    .withSyncPolicy(syncPolicy);

  return hdfsBolt;
}
 
Example 10
Project: storm-topology-examples   File: ConfigureHdfsBolt.java   Source Code and License 5 votes vote down vote up
public static void configureHdfsBolt(TopologyBuilder builder, 
                                     String delimiter, 
                                     String outputPath, 
                                     String hdfsUri,
                                     String hdfsBoltName, 
                                     String spoutName,
                                     int parallelismHint,
                                     FileRotationPolicy rotationPolicy,
                                     int syncCount) {
    
    LOG.info("HDFSBOLT: Configuring the HdfsBolt");
    
    // Define the RecordFormat, SyncPolicy, and FileNameFormat
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(delimiter);
    SyncPolicy syncPolicy = new CountSyncPolicy(syncCount);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(outputPath);
    
    // Configure the Bolt
    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(hdfsUri)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
    
    // Set the Bolt
    builder.setBolt(hdfsBoltName, bolt, parallelismHint).shuffleGrouping(spoutName);

}
 
Example 11
Project: storm-topology-examples   File: ConfigureHdfsBolt.java   Source Code and License 5 votes vote down vote up
public static FileRotationPolicy configureFileRotationPolicy(String propsFileName) throws IOException {
    // Parse the properties file
    PropertyParser propertyParser = new PropertyParser();
    propertyParser.parsePropsFile(propsFileName);
    
    // Get the config value to determine which FileRotationPolicy is enabled
    boolean useTimeBasedFileRotationPolicy = 
            Boolean.parseBoolean(propertyParser.getProperty(
                    ConfigVars.HDFS_BOLT_USE_TIME_BASED_FILEROTATIONPOLICY_KEY));

    boolean useSizeBasedFileRotationPolicy =
            Boolean.parseBoolean(propertyParser.getProperty(
                    ConfigVars.HDFS_BOLT_USE_SIZE_BASED_FILEROTATIONPOLICY_KEY));
    
    // Guard against both size and time based policies being enabled
    if (useSizeBasedFileRotationPolicy && useTimeBasedFileRotationPolicy) {
        LOG.error("ERROR: You cannot use both time and size based rotation policies");
        LOG.error("ERROR: Validate that either hdfs.bolt.use.time.based.filerotationpolicy" +
                " or hdfs.bolt.use.size.based.filerotationpolicy is false");
        throw new IllegalArgumentException("ERROR: You cannot use both time and size based rotation policies");
    }
    
    
    // Return the appropriate FileRotationPolicy based on the config
    if (useSizeBasedFileRotationPolicy) {
        return getSizeBasedFileRotationPolicy(
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_USE_SIZE_BASED_FILEROTATIONPOLICY_UNIT_KEY),
                Integer.parseInt(propertyParser.getProperty(
                        ConfigVars.HDFS_BOLT_USE_SIZE_BASED_FILEROTATIONPOLICY_SIZE_KEY)));
    } else {
        return getTimeBasedFileRotationPolicy(
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_USE_TIME_BASED_FILEROTATIONPOLICY_UNIT_KEY),
                Integer.parseInt(propertyParser.getProperty(
                        ConfigVars.HDFS_BOLT_USE_TIME_BASED_FILEROTATIONPOLICY_DURATION_KEY)));
    }
    
}
 
Example 12
Project: metron   File: SourceHandler.java   Source Code and License 5 votes vote down vote up
public SourceHandler(List<RotationAction> rotationActions
                    , FileRotationPolicy rotationPolicy
                    , SyncPolicy syncPolicy
                    , FileNameFormat fileNameFormat
                    , SourceHandlerCallback cleanupCallback) throws IOException {
  this.rotationActions = rotationActions;
  this.rotationPolicy = rotationPolicy;
  this.syncPolicy = syncPolicy;
  this.fileNameFormat = fileNameFormat;
  this.cleanupCallback = cleanupCallback;
  initialize();
}
 
Example 13
Project: storm-kafka-hdfs-starter   File: ConfigureHdfsBolt.java   Source Code and License 5 votes vote down vote up
public static void configureHdfsBolt(TopologyBuilder builder, String delimiter, String outputPath, String hdfsUri) {
    RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(delimiter);
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);
    //FileRotationPolicy rotationPolicy = new TimedRotationPolicy(300, TimedRotationPolicy.TimeUnit.SECONDS);
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(1, FileSizeRotationPolicy.Units.KB);
    FileNameFormat fileNameFormat = new DefaultFileNameFormat().withPath(outputPath);
    HdfsBolt bolt = new HdfsBolt()
            .withFsUrl(hdfsUri)
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy);
    builder.setBolt("hdfsbolt", bolt, 1).shuffleGrouping("kafkaspout");

}
 
Example 14
Project: storm-kafka-examples   File: HdfsTopology.java   Source Code and License 4 votes vote down vote up
public static void main(String[] args) {
    try{
        String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181";
        String topic = "order";
        String groupId = "id";
        int spoutNum = 3;
        int boltNum = 1;
        ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper
        SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId);  // create /order /id
        spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme());
        KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig);

        // HDFS bolt
        // use "|" instead of "," for field delimiter
        RecordFormat format = new DelimitedRecordFormat()
                .withFieldDelimiter("|");

        // sync the filesystem after every 1k tuples
        SyncPolicy syncPolicy = new CountSyncPolicy(1000);

        // rotate files when they reach 5MB
        FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB);
        // FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

        FileNameFormat fileNameFormat = new DefaultFileNameFormat()
                .withPath("/tmp/").withPrefix("order_").withExtension(".log");

        HdfsBolt hdfsBolt = new HdfsBolt()
                .withFsUrl("hdfs://wxb-1:8020")
                .withFileNameFormat(fileNameFormat)
                .withRecordFormat(format)
                .withRotationPolicy(rotationPolicy)
                .withSyncPolicy(syncPolicy);

        TopologyBuilder builder = new TopologyBuilder();
        builder.setSpout("spout", kafkaSpout, spoutNum);
        builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout");
        builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check");
        builder.setBolt("hdfs", hdfsBolt,boltNum).shuffleGrouping("counter");

        Config config = new Config();
        config.setDebug(true);

        if(args!=null && args.length > 0) {
            config.setNumWorkers(2);
            StormSubmitter.submitTopology(args[0], config, builder.createTopology());
        } else {
            config.setMaxTaskParallelism(2);

            LocalCluster cluster = new LocalCluster();
            cluster.submitTopology("Wordcount-Topology", config, builder.createTopology());

            Thread.sleep(500000);

            cluster.shutdown();
        }
    }catch (Exception e) {
        e.printStackTrace();
    }
}
 
Example 15
Project: storm-kafka-hdfs-example   File: HdfsBoltConfigBuilder.java   Source Code and License 4 votes vote down vote up
public FileRotationPolicy getFileRotationPolicy() {
    return fileRotationPolicy;
}
 
Example 16
Project: storm-kafka-hdfs-example   File: HdfsBoltConfigBuilder.java   Source Code and License 4 votes vote down vote up
public Builder setFileRotationPolicy(FileRotationPolicy fileRotationPolicy) {
    this.fileRotationPolicy = fileRotationPolicy;
    return this;
}
 
Example 17
Project: storm-kafka-hdfs-example   File: KafkaHdfsTopology.java   Source Code and License 4 votes vote down vote up
public static void main(String[] args) throws Exception {

        if (args.length < 1) {
            System.out.println("USAGE: storm jar </path/to/topo.jar> <com.package.TopologyMainClass> " +
                    "</path/to/config.properties>");
            System.exit(1);
        }
        String propFilePath = args[0];

        // Setup the property parser
        PropertyParser propertyParser = new PropertyParser(propFilePath);
        propertyParser.parsePropsFile();

        // Create the topology builder object
        TopologyBuilder topologyBuilder = new TopologyBuilder();

        // Setup the Kafka Spout
        KafkaSpoutConfigBuilder kafkaSpoutConfigBuilder = new KafkaSpoutConfigBuilder.Builder()
                .setZookeeperConnectionString(propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY))
                .setKafkaTopic(propertyParser.getProperty(ConfigVars.KAFKA_TOPIC_KEY))
                .setKafkaStartOffset(propertyParser.getProperty(ConfigVars.KAFKA_START_OFFSET_KEY))
                .setSpoutName(propertyParser.getProperty(ConfigVars.STORM_KAFKA_SPOUT_NAME_KEY))
                .setSpoutParallelismHint(Integer.parseInt(
                        propertyParser.getProperty(ConfigVars.STORM_KAFKA_SPOUT_PARALLELISM_KEY)))
                .setSpoutSchemeClass(propertyParser.getProperty(ConfigVars.STORM_KAFKA_SPOUT_SCHEME_CLASS_KEY))
                .build();


        // Add the spout to the topology
        topologyBuilder.setSpout(kafkaSpoutConfigBuilder.getSpoutName(), kafkaSpoutConfigBuilder.getKafkaSpout(),
                kafkaSpoutConfigBuilder.getSpoutParallelismHint());


        // Setup the HDFS Bolt
        FileRotationPolicy fileRotationPolicy = HdfsBoltConfigBuilder.getTimeBasedFileRotationPolicy(
                propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_ROTATION_POLICY_UNITS_KEY),
                Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_ROTATION_POLICY_COUNT_KEY)));
        HdfsBoltConfigBuilder hdfsBoltConfigBuilder = new HdfsBoltConfigBuilder.Builder()
                .setFieldDelimiter(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_FIELD_DELIMITER_KEY))
                .setOutputLocation(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_OUTPUT_LOCATION_KEY))
                .setHdfsDefaultFs(propertyParser.getProperty(ConfigVars.HDFS_DEFAULT_FS_KEY))
                .setSyncCount(Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_SYNC_COUNT_KEY)))
                .setBoltParallelism(Integer.parseInt(
                        propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_PARALLELISM_KEY)))
                .setBoltName(propertyParser.getProperty(ConfigVars.STORM_HDFS_BOLT_NAME_KEY))
                .setFileRotationPolicy(fileRotationPolicy)
                .build();


        // Add the HDFS Bolt to the topology
        topologyBuilder.setBolt(hdfsBoltConfigBuilder.getBoltName(), hdfsBoltConfigBuilder.getHdfsBolt(),
                hdfsBoltConfigBuilder.getBoltParallelism()).shuffleGrouping(kafkaSpoutConfigBuilder.getSpoutName());


        // Storm Topology Config
        Config stormConfig = StormConfig.createStormConfig(
                Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.STORM_ENABLE_DEBUG_KEY)),
                Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_NUM_WORKERS_KEY)));

        // Submit the topology
        StormSubmitter.submitTopologyWithProgressBar(propertyParser.getProperty(ConfigVars.STORM_TOPOLOGY_NAME_KEY),
                stormConfig, topologyBuilder.createTopology());

    }
 
Example 18
Project: storm-topology-examples   File: KafkaHdfsTopology.java   Source Code and License 4 votes vote down vote up
public static void main(String[] args) throws Exception {

        if (args.length < 1) {
            System.out.println("USAGE: storm jar </path/to/topo.jar> <com.package.TopologyMainClass> " +
                    "</path/to/config.properties>");
            System.exit(1);
        }
        String propFilePath = args[0];

        // Parse the properties file
        PropertyParser propertyParser = new PropertyParser();
        propertyParser.parsePropsFile(propFilePath);

        TopologyBuilder builder = new TopologyBuilder();

        // Setup the Kafka Spout
        ConfigureKafkaSpout.configureKafkaSpout(builder,
                propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY),
                propertyParser.getProperty(ConfigVars.KAFKA_TOPIC_KEY),
                propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_START_OFFSET_KEY),
                Integer.parseInt(propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_PARALLELISM_KEY)),
                propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_NAME_KEY),
                propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_SCHEME_CLASS_KEY));


        // Configure the HdfsBolt
        FileRotationPolicy fileRotationPolicy = ConfigureHdfsBolt.configureFileRotationPolicy(propFilePath);
        ConfigureHdfsBolt.configureHdfsBolt(builder,
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_FIELD_DELIMITER_KEY),
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_OUTPUT_LOCATION_KEY),
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_DFS_URI_KEY),
                propertyParser.getProperty(ConfigVars.HDFS_BOLT_NAME_KEY),
                propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_NAME_KEY),
                Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_BOLT_PARALLELISM_KEY)),
                fileRotationPolicy,
                Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_BOLT_SYNC_COUNT_KEY)));

        // Storm Topology Config
        Config stormConfig = StormConfig.createStormConfig(
                Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.STORM_ENABLE_DEBUG_KEY)),
                Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_NUM_WORKERS_KEY)));

        // Submit the topology
        StormSubmitter.submitTopologyWithProgressBar(propertyParser.getProperty(ConfigVars.STORM_TOPOLOGY_NAME_KEY),
                stormConfig, builder.createTopology());

    }
 
Example 19
Project: storm-topology-examples   File: KafkaHiveHdfsTopologyTest.java   Source Code and License 4 votes vote down vote up
public void runStormKafkaHiveHdfsTopology() throws IOException {
    LOG.info("STORM: Starting Topology: " + propertyParser.getProperty(ConfigVars.STORM_TOPOLOGY_NAME_KEY));
    TopologyBuilder builder = new TopologyBuilder();

    // Configure the KafkaSpout
    ConfigureKafkaSpout.configureKafkaSpout(builder,
            propertyParser.getProperty(ConfigVars.ZOOKEEPER_CONNECTION_STRING_KEY),
            propertyParser.getProperty(ConfigVars.KAFKA_TOPIC_KEY),
            propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_START_OFFSET_KEY),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_PARALLELISM_KEY)),
            propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_NAME_KEY),
            propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_SCHEME_CLASS_KEY));

    // Configure the HdfsBolt
    FileRotationPolicy fileRotationPolicy = ConfigureHdfsBolt.configureFileRotationPolicy(PROP_FILE);
    ConfigureHdfsBolt.configureHdfsBolt(builder,
            propertyParser.getProperty(ConfigVars.HDFS_BOLT_FIELD_DELIMITER_KEY),
            propertyParser.getProperty(ConfigVars.HDFS_BOLT_OUTPUT_LOCATION_KEY),
            propertyParser.getProperty(ConfigVars.HDFS_BOLT_DFS_URI_KEY),
            propertyParser.getProperty(ConfigVars.HDFS_BOLT_NAME_KEY),
            propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_NAME_KEY),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_BOLT_PARALLELISM_KEY)),
            fileRotationPolicy,
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HDFS_BOLT_SYNC_COUNT_KEY)));

    // Configure the HiveBolt
    ConfigureHiveBolt.configureHiveStreamingBolt(builder,
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_COLUMN_LIST_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_PARTITION_LIST_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_COLUMN_PARTITION_LIST_DELIMITER_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_METASTORE_URI_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_DATABASE_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_TABLE_KEY),
            propertyParser.getProperty(ConfigVars.HIVE_BOLT_NAME_KEY),
            propertyParser.getProperty(ConfigVars.KAFKA_SPOUT_NAME_KEY),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_PARALLELISM_KEY)),
            Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.HIVE_BOLT_AUTO_CREATE_PARTITIONS_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_TXNS_PER_BATCH_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_MAX_OPEN_CONNECTIONS_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_BATCH_SIZE_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_IDLE_TIMEOUT_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.HIVE_BOLT_HEARTBEAT_INTERVAL_KEY)));


    // Storm Topology Config
    Config stormConfig = StormConfig.createStormConfig(
            Boolean.parseBoolean(propertyParser.getProperty(ConfigVars.STORM_ENABLE_DEBUG_KEY)),
            Integer.parseInt(propertyParser.getProperty(ConfigVars.STORM_NUM_WORKERS_KEY)));

    // Submit the topology
    stormLocalCluster.submitTopology(propertyParser.getProperty(ConfigVars.STORM_TOPOLOGY_NAME_KEY), new Config(), builder.createTopology());
}
 
Example 20
Project: metron   File: HdfsWriter.java   Source Code and License 4 votes vote down vote up
public HdfsWriter withRotationPolicy(FileRotationPolicy rotationPolicy){
  this.rotationPolicy = rotationPolicy;
  return this;
}
 
Example 21
Project: opensoc-streaming   File: TopologyRunner.java   Source Code and License 4 votes vote down vote up
private boolean initializeHDFSBolt(String topology_name, String name) {
	try {

		String messageUpstreamComponent = messageComponents
				.get(messageComponents.size() - 1);

		System.out.println("[OpenSOC] ------" + name
				+ " is initializing from " + messageUpstreamComponent);

		RecordFormat format = new DelimitedRecordFormat()
				.withFieldDelimiter(
						config.getString("bolt.hdfs.field.delimiter")
								.toString()).withFields(
						new Fields("message"));

		// sync the file system after every x number of tuples
		SyncPolicy syncPolicy = new CountSyncPolicy(Integer.valueOf(config
				.getString("bolt.hdfs.batch.size").toString()));

		// rotate files when they reach certain size
		FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(
				Float.valueOf(config.getString(
						"bolt.hdfs.file.rotation.size.in.mb").toString()),
				Units.MB);

		FileNameFormat fileNameFormat = new DefaultFileNameFormat()
				.withPath(config.getString("bolt.hdfs.wip.file.path")
						.toString());

		// Post rotate action
		MoveFileAction moveFileAction = (new MoveFileAction())
				.toDestination(config.getString(
						"bolt.hdfs.finished.file.path").toString());

		HdfsBolt hdfsBolt = new HdfsBolt()
				.withFsUrl(
						config.getString("bolt.hdfs.file.system.url")
								.toString())
				.withFileNameFormat(fileNameFormat)
				.withRecordFormat(format)
				.withRotationPolicy(rotationPolicy)
				.withSyncPolicy(syncPolicy)
				.addRotationAction(moveFileAction);
		if (config.getString("bolt.hdfs.compression.codec.class") != null) {
			hdfsBolt.withCompressionCodec(config.getString(
					"bolt.hdfs.compression.codec.class").toString());
		}

		builder.setBolt(name, hdfsBolt,
				config.getInt("bolt.hdfs.parallelism.hint"))
				.shuffleGrouping(messageUpstreamComponent, "message")
				.setNumTasks(config.getInt("bolt.hdfs.num.tasks"));

	} catch (Exception e) {
		e.printStackTrace();
		System.exit(0);
	}

	return true;
}
 
Example 22
Project: storm-hdfs   File: HdfsBolt.java   Source Code and License 4 votes vote down vote up
public HdfsBolt withRotationPolicy(FileRotationPolicy rotationPolicy){
    this.rotationPolicy = rotationPolicy;
    return this;
}
 
Example 23
Project: storm-hdfs   File: SequenceFileBolt.java   Source Code and License 4 votes vote down vote up
public SequenceFileBolt withRotationPolicy(FileRotationPolicy rotationPolicy) {
    this.rotationPolicy = rotationPolicy;
    return this;
}
 
Example 24
Project: storm-hdfs   File: SequenceFileTopology.java   Source Code and License 4 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);

    SentenceSpout spout = new SentenceSpout();

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/source/")
            .withExtension(".seq");

    // create sequence format instance.
    DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence");

    SequenceFileBolt bolt = new SequenceFileBolt()
            .withFsUrl(args[0])
            .withFileNameFormat(fileNameFormat)
            .withSequenceFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy)
            .withCompressionType(SequenceFile.CompressionType.RECORD)
            .withCompressionCodec("deflate")
            .addRotationAction(new MoveFileAction().toDestination("/dest/"));




    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4)
            .shuffleGrouping(SENTENCE_SPOUT_ID);


    if (args.length == 1) {
        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(120);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
        System.exit(0);
    } else if(args.length == 2) {
        StormSubmitter.submitTopology(args[1], config, builder.createTopology());
    }
}
 
Example 25
Project: storm-hdfs   File: HdfsFileTopology.java   Source Code and License 4 votes vote down vote up
public static void main(String[] args) throws Exception {
    Config config = new Config();
    config.setNumWorkers(1);

    SentenceSpout spout = new SentenceSpout();

    // sync the filesystem after every 1k tuples
    SyncPolicy syncPolicy = new CountSyncPolicy(1000);

    // rotate files when they reach 5MB
    FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES);

    FileNameFormat fileNameFormat = new DefaultFileNameFormat()
            .withPath("/foo/")
            .withExtension(".txt");



    // use "|" instead of "," for field delimiter
    RecordFormat format = new DelimitedRecordFormat()
            .withFieldDelimiter("|");

    Yaml yaml = new Yaml();
    InputStream in = new FileInputStream(args[1]);
    Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in);
    in.close();
    config.put("hdfs.config", yamlConf);

    HdfsBolt bolt = new HdfsBolt()
            .withConfigKey("hdfs.config")
            .withFsUrl(args[0])
            .withFileNameFormat(fileNameFormat)
            .withRecordFormat(format)
            .withRotationPolicy(rotationPolicy)
            .withSyncPolicy(syncPolicy)
            .addRotationAction(new MoveFileAction().toDestination("/dest2/"));

    TopologyBuilder builder = new TopologyBuilder();

    builder.setSpout(SENTENCE_SPOUT_ID, spout, 1);
    // SentenceSpout --> MyBolt
    builder.setBolt(BOLT_ID, bolt, 4)
            .shuffleGrouping(SENTENCE_SPOUT_ID);

    if (args.length == 2) {
        LocalCluster cluster = new LocalCluster();

        cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology());
        waitForSeconds(120);
        cluster.killTopology(TOPOLOGY_NAME);
        cluster.shutdown();
        System.exit(0);
    } else if (args.length == 3) {
        StormSubmitter.submitTopology(args[0], config, builder.createTopology());
    } else{
        System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>");
    }
}
 
Example 26
Project: storm-crawler   File: GzipHdfsBolt.java   Source Code and License 4 votes vote down vote up
public GzipHdfsBolt withRotationPolicy(FileRotationPolicy rotationPolicy) {
    this.rotationPolicy = rotationPolicy;
    return this;
}