org.apache.storm.hdfs.bolt.sync.SyncPolicy Java Examples
The following examples show how to use
org.apache.storm.hdfs.bolt.sync.SyncPolicy.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ClonedSyncPolicyCreator.java From metron with Apache License 2.0 | 6 votes |
@Override public SyncPolicy create(String sensor, WriterConfiguration config) { try { //we do a deep clone of the SyncPolicy via kryo serialization. This gives us a fresh policy //to work with. The reason we need a fresh policy is that we want to ensure each handler //(one handler per task & sensor type and one handler per file) has its own sync policy. // Reusing a sync policy is a bad idea, so we need to clone it here. Unfortunately the // SyncPolicy object does not implement Cloneable, so we'll need to clone it via serialization //to get a fresh policy object. Note: this would be expensive if it was in the critical path, // but should be called infrequently (once per sync). // Reset the SyncPolicy to ensure that the new count properly resets. syncPolicy.reset(); byte[] serializedForm = SerDeUtils.toBytes(syncPolicy); return SerDeUtils.fromBytes(serializedForm, SyncPolicy.class); } catch (Exception e) { throw new IllegalStateException(e.getMessage(), e); } }
Example #2
Source File: MovingAvgLocalTopologyRunner.java From hadoop-arch-book with Apache License 2.0 | 6 votes |
/** * Create bolt which will persist ticks to HDFS. */ private static HdfsBolt createHdfsBolt() { // Use "|" instead of "," for field delimiter: RecordFormat format = new DelimitedRecordFormat() .withFieldDelimiter("|"); // sync the filesystem after every 1k tuples: SyncPolicy syncPolicy = new CountSyncPolicy(100); // Rotate files when they reach 5MB: FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB); // Write records to <user>/stock-ticks/ directory in HDFS: FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("stock-ticks/"); HdfsBolt hdfsBolt = new HdfsBolt() .withFsUrl("hdfs://localhost:8020") .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy); return hdfsBolt; }
Example #3
Source File: SourceHandler.java From metron with Apache License 2.0 | 5 votes |
public SourceHandler(List<RotationAction> rotationActions , FileRotationPolicy rotationPolicy , SyncPolicy syncPolicy , FileNameFormat fileNameFormat , SourceHandlerCallback cleanupCallback) throws IOException { this.rotationActions = rotationActions; this.rotationPolicy = rotationPolicy; this.syncPolicy = syncPolicy; this.fileNameFormat = fileNameFormat; this.cleanupCallback = cleanupCallback; initialize(); }
Example #4
Source File: ClonedSyncPolicyCreatorTest.java From metron with Apache License 2.0 | 5 votes |
@Test public void testClonedPolicy() { CountSyncPolicy basePolicy = new CountSyncPolicy(5); ClonedSyncPolicyCreator creator = new ClonedSyncPolicyCreator(basePolicy); //ensure cloned policy continues to work and adheres to the contract: mark on 5th call. SyncPolicy clonedPolicy = creator.create("blah", null); for(int i = 0;i < 4;++i) { assertFalse(clonedPolicy.mark(null, i)); } assertTrue(clonedPolicy.mark(null, 5)); //reclone policy and ensure it adheres to the original contract. clonedPolicy = creator.create("blah", null); assertFalse(clonedPolicy.mark(null, 0)); }
Example #5
Source File: HdfsTopology.java From storm-kafka-examples with Apache License 2.0 | 4 votes |
public static void main(String[] args) { try{ String zkhost = "wxb-1:2181,wxb-2:2181,wxb-3:2181"; String topic = "order"; String groupId = "id"; int spoutNum = 3; int boltNum = 1; ZkHosts zkHosts = new ZkHosts(zkhost);//kafaka所在的zookeeper SpoutConfig spoutConfig = new SpoutConfig(zkHosts, topic, "/order", groupId); // create /order /id spoutConfig.scheme = new SchemeAsMultiScheme(new StringScheme()); KafkaSpout kafkaSpout = new KafkaSpout(spoutConfig); // HDFS bolt // use "|" instead of "," for field delimiter RecordFormat format = new DelimitedRecordFormat() .withFieldDelimiter("|"); // sync the filesystem after every 1k tuples SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, FileSizeRotationPolicy.Units.MB); // FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/tmp/").withPrefix("order_").withExtension(".log"); HdfsBolt hdfsBolt = new HdfsBolt() .withFsUrl("hdfs://wxb-1:8020") .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout("spout", kafkaSpout, spoutNum); builder.setBolt("check", new CheckOrderBolt(), boltNum).shuffleGrouping("spout"); builder.setBolt("counter", new CounterBolt(),boltNum).shuffleGrouping("check"); builder.setBolt("hdfs", hdfsBolt,boltNum).shuffleGrouping("counter"); Config config = new Config(); config.setDebug(true); if(args!=null && args.length > 0) { config.setNumWorkers(2); StormSubmitter.submitTopology(args[0], config, builder.createTopology()); } else { config.setMaxTaskParallelism(2); LocalCluster cluster = new LocalCluster(); cluster.submitTopology("Wordcount-Topology", config, builder.createTopology()); Thread.sleep(500000); cluster.shutdown(); } }catch (Exception e) { e.printStackTrace(); } }
Example #6
Source File: HdfsWriter.java From metron with Apache License 2.0 | 4 votes |
public HdfsWriter withSyncPolicy(SyncPolicy syncPolicy){ this.syncPolicy = syncPolicy; return this; }
Example #7
Source File: ClonedSyncPolicyCreator.java From metron with Apache License 2.0 | 4 votes |
public ClonedSyncPolicyCreator(SyncPolicy policy) { syncPolicy = policy; }
Example #8
Source File: TopologyRunner.java From opensoc-streaming with Apache License 2.0 | 4 votes |
private boolean initializeHDFSBolt(String topology_name, String name) { try { String messageUpstreamComponent = messageComponents .get(messageComponents.size() - 1); System.out.println("[OpenSOC] ------" + name + " is initializing from " + messageUpstreamComponent); RecordFormat format = new DelimitedRecordFormat() .withFieldDelimiter( config.getString("bolt.hdfs.field.delimiter") .toString()).withFields( new Fields("message")); // sync the file system after every x number of tuples SyncPolicy syncPolicy = new CountSyncPolicy(Integer.valueOf(config .getString("bolt.hdfs.batch.size").toString())); // rotate files when they reach certain size FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy( Float.valueOf(config.getString( "bolt.hdfs.file.rotation.size.in.mb").toString()), Units.MB); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath(config.getString("bolt.hdfs.wip.file.path") .toString()); // Post rotate action MoveFileAction moveFileAction = (new MoveFileAction()) .toDestination(config.getString( "bolt.hdfs.finished.file.path").toString()); HdfsBolt hdfsBolt = new HdfsBolt() .withFsUrl( config.getString("bolt.hdfs.file.system.url") .toString()) .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy) .addRotationAction(moveFileAction); if (config.getString("bolt.hdfs.compression.codec.class") != null) { hdfsBolt.withCompressionCodec(config.getString( "bolt.hdfs.compression.codec.class").toString()); } builder.setBolt(name, hdfsBolt, config.getInt("bolt.hdfs.parallelism.hint")) .shuffleGrouping(messageUpstreamComponent, "message") .setNumTasks(config.getInt("bolt.hdfs.num.tasks")); } catch (Exception e) { e.printStackTrace(); System.exit(0); } return true; }
Example #9
Source File: HdfsBolt.java From storm-hdfs with Apache License 2.0 | 4 votes |
public HdfsBolt withSyncPolicy(SyncPolicy syncPolicy){ this.syncPolicy = syncPolicy; return this; }
Example #10
Source File: SequenceFileBolt.java From storm-hdfs with Apache License 2.0 | 4 votes |
public SequenceFileBolt withSyncPolicy(SyncPolicy syncPolicy) { this.syncPolicy = syncPolicy; return this; }
Example #11
Source File: SequenceFileTopology.java From storm-hdfs with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); config.setNumWorkers(1); SentenceSpout spout = new SentenceSpout(); // sync the filesystem after every 1k tuples SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/source/") .withExtension(".seq"); // create sequence format instance. DefaultSequenceFormat format = new DefaultSequenceFormat("timestamp", "sentence"); SequenceFileBolt bolt = new SequenceFileBolt() .withFsUrl(args[0]) .withFileNameFormat(fileNameFormat) .withSequenceFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy) .withCompressionType(SequenceFile.CompressionType.RECORD) .withCompressionCodec("deflate") .addRotationAction(new MoveFileAction().toDestination("/dest/")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 1); // SentenceSpout --> MyBolt builder.setBolt(BOLT_ID, bolt, 4) .shuffleGrouping(SENTENCE_SPOUT_ID); if (args.length == 1) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(120); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); System.exit(0); } else if(args.length == 2) { StormSubmitter.submitTopology(args[1], config, builder.createTopology()); } }
Example #12
Source File: HdfsFileTopology.java From storm-hdfs with Apache License 2.0 | 4 votes |
public static void main(String[] args) throws Exception { Config config = new Config(); config.setNumWorkers(1); SentenceSpout spout = new SentenceSpout(); // sync the filesystem after every 1k tuples SyncPolicy syncPolicy = new CountSyncPolicy(1000); // rotate files when they reach 5MB FileRotationPolicy rotationPolicy = new TimedRotationPolicy(1.0f, TimedRotationPolicy.TimeUnit.MINUTES); FileNameFormat fileNameFormat = new DefaultFileNameFormat() .withPath("/foo/") .withExtension(".txt"); // use "|" instead of "," for field delimiter RecordFormat format = new DelimitedRecordFormat() .withFieldDelimiter("|"); Yaml yaml = new Yaml(); InputStream in = new FileInputStream(args[1]); Map<String, Object> yamlConf = (Map<String, Object>) yaml.load(in); in.close(); config.put("hdfs.config", yamlConf); HdfsBolt bolt = new HdfsBolt() .withConfigKey("hdfs.config") .withFsUrl(args[0]) .withFileNameFormat(fileNameFormat) .withRecordFormat(format) .withRotationPolicy(rotationPolicy) .withSyncPolicy(syncPolicy) .addRotationAction(new MoveFileAction().toDestination("/dest2/")); TopologyBuilder builder = new TopologyBuilder(); builder.setSpout(SENTENCE_SPOUT_ID, spout, 1); // SentenceSpout --> MyBolt builder.setBolt(BOLT_ID, bolt, 4) .shuffleGrouping(SENTENCE_SPOUT_ID); if (args.length == 2) { LocalCluster cluster = new LocalCluster(); cluster.submitTopology(TOPOLOGY_NAME, config, builder.createTopology()); waitForSeconds(120); cluster.killTopology(TOPOLOGY_NAME); cluster.shutdown(); System.exit(0); } else if (args.length == 3) { StormSubmitter.submitTopology(args[0], config, builder.createTopology()); } else{ System.out.println("Usage: HdfsFileTopology [topology name] <yaml config file>"); } }
Example #13
Source File: GzipHdfsBolt.java From storm-crawler with Apache License 2.0 | 4 votes |
public GzipHdfsBolt withSyncPolicy(SyncPolicy syncPolicy) { this.syncPolicy = syncPolicy; return this; }
Example #14
Source File: SyncPolicyCreator.java From metron with Apache License 2.0 | votes |
SyncPolicy create(String sensor, WriterConfiguration config);