org.apache.flink.api.java.utils.ParameterTool Java Examples

The following examples show how to use org.apache.flink.api.java.utils.ParameterTool. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: AvroDataGeneratorJob.java    From flink-tutorials with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	ParameterTool params = Utils.parseArgs(args);
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	KafkaSerializationSchema<Message> schema = ClouderaRegistryKafkaSerializationSchema.<Message>
			builder(params.getRequired(K_KAFKA_TOPIC))
			.setConfig(Utils.readSchemaRegistryProperties(params))
			.setKey(Message::getId)
			.build();

	FlinkKafkaProducer<Message> kafkaSink = new FlinkKafkaProducer<>(
			"default", schema, Utils.readKafkaProperties(params), FlinkKafkaProducer.Semantic.AT_LEAST_ONCE);

	DataStream<Message> input = env.addSource(new DataGeneratorSource()).name("Data Generator Source");

	input.addSink(kafkaSink)
			.name("Kafka Sink")
			.uid("Kafka Sink");

	input.print();

	env.execute("Data Generator Job");
}
 
Example #2
Source File: StreamWordCount.java    From jstorm with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
        StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

//        env.addS
        ParameterTool tool = ParameterTool.fromArgs(args);
        int sourceParallel = Integer.parseInt(tool.get("s"));
        int operatorParallel = Integer.parseInt(tool.get("p"));

        System.out.println("sourceParallel: " + sourceParallel + ", operatorParallel: " + operatorParallel);

        env.setParallelism(operatorParallel);

        // get input data
        DataStream<String> text = env.addSource(new WordSource()).setParallelism(sourceParallel);

        DataStream<Tuple2<String, Integer>> counts =
                // split up the lines in pairs (2-tuples) containing: (word,1)
                text.flatMap(new LineSplitter())
                        // group by the tuple field "0" and sum up tuple field "1"
                        .keyBy(0)
                        .sum(1);

        // execute program
//        env.execute("StreamWordCount");
        System.out.println(env.getExecutionPlan());
    }
 
Example #3
Source File: RideCleansingSolution.java    From flink-training-exercises with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {

		ParameterTool params = ParameterTool.fromArgs(args);
		final String input = params.get("input", pathToRideData);

		final int maxEventDelay = 60;       // events are out of order by max 60 seconds
		final int servingSpeedFactor = 600; // events of 10 minutes are served in 1 second

		// set up streaming execution environment
		StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
		env.setParallelism(ExerciseBase.parallelism);

		// start the data generator
		DataStream<TaxiRide> rides = env.addSource(rideSourceOrTest(new TaxiRideSource(input, maxEventDelay, servingSpeedFactor)));

		DataStream<TaxiRide> filteredRides = rides
				// keep only those rides and both start and end in NYC
				.filter(new NYCFilter());

		// print the filtered stream
		printOrTest(filteredRides);

		// run the cleansing pipeline
		env.execute("Taxi Ride Cleansing");
	}
 
Example #4
Source File: DataSetFineGrainedRecoveryTestProgram.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	final ParameterTool params = ParameterTool.fromArgs(args);
	final String latchFilePath = params.getRequired("latchFilePath");
	final String outputPath = params.getRequired("outputPath");

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().setExecutionMode(ExecutionMode.BATCH_FORCED);
	env.setParallelism(4);

	env.generateSequence(0, 1000)
		.map(new BlockingIncrementingMapFunction(latchFilePath))
		.writeAsText(outputPath, FileSystem.WriteMode.OVERWRITE)
		.setParallelism(1);

	env.execute();
}
 
Example #5
Source File: FlinkKafkaConsumerTest1.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
        final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
        StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);
        Properties props = buildKafkaProps(parameterTool);
        //kafka topic list
        List<String> topics = Arrays.asList(parameterTool.get("metrics.topic"), parameterTool.get("logs.topic"));
        FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(topics, new MetricSchema(), props);
        //kafka topic Pattern
        //FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(java.utils.regex.Pattern.compile("test-topic-[0-9]"), new MetricSchema(), props);


//        consumer.setStartFromLatest();
//        consumer.setStartFromEarliest()
        DataStreamSource<MetricEvent> data = env.addSource(consumer);

        data.print();

        env.execute("flink kafka connector test");
    }
 
Example #6
Source File: DataStreamAllroundTestJobFactory.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
static SourceFunction<Event> createEventSource(ParameterTool pt) {
	return new SequenceGeneratorSource(
		pt.getInt(
			SEQUENCE_GENERATOR_SRC_KEYSPACE.key(),
			SEQUENCE_GENERATOR_SRC_KEYSPACE.defaultValue()),
		pt.getInt(
			SEQUENCE_GENERATOR_SRC_PAYLOAD_SIZE.key(),
			SEQUENCE_GENERATOR_SRC_PAYLOAD_SIZE.defaultValue()),
		pt.getLong(
			SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.key(),
			SEQUENCE_GENERATOR_SRC_EVENT_TIME_MAX_OUT_OF_ORDERNESS.defaultValue()),
		pt.getLong(
			SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.key(),
			SEQUENCE_GENERATOR_SRC_EVENT_TIME_CLOCK_PROGRESS.defaultValue()),
		pt.getLong(
			SEQUENCE_GENERATOR_SRC_SLEEP_TIME.key(),
			SEQUENCE_GENERATOR_SRC_SLEEP_TIME.defaultValue()),
		pt.getLong(
			SEQUENCE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(),
			SEQUENCE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue()));
}
 
Example #7
Source File: Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
private static void writeEventToHbase(String string, ParameterTool parameterTool) throws IOException {
    Configuration configuration = HBaseConfiguration.create();
    configuration.set(HBASE_ZOOKEEPER_QUORUM, parameterTool.get(HBASE_ZOOKEEPER_QUORUM));
    configuration.set(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT, parameterTool.get(HBASE_ZOOKEEPER_PROPERTY_CLIENTPORT));
    configuration.set(HBASE_RPC_TIMEOUT, parameterTool.get(HBASE_RPC_TIMEOUT));
    configuration.set(HBASE_CLIENT_OPERATION_TIMEOUT, parameterTool.get(HBASE_CLIENT_OPERATION_TIMEOUT));
    configuration.set(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, parameterTool.get(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD));

    Connection connect = ConnectionFactory.createConnection(configuration);
    Admin admin = connect.getAdmin();
    if (!admin.tableExists(HBASE_TABLE_NAME)) { //检查是否有该表,如果没有,创建
        admin.createTable(new HTableDescriptor(HBASE_TABLE_NAME).addFamily(new HColumnDescriptor(INFO_STREAM)));
    }
    Table table = connect.getTable(HBASE_TABLE_NAME);
    TimeStamp ts = new TimeStamp(new Date());
    Date date = ts.getDate();
    Put put = new Put(Bytes.toBytes(date.getTime()));
    put.addColumn(Bytes.toBytes(INFO_STREAM), Bytes.toBytes("test"), Bytes.toBytes(string));
    table.put(put);
    table.close();
    connect.close();
}
 
Example #8
Source File: FailureRateRestartStrategyMain.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.getConfig().setGlobalJobParameters(ParameterTool.fromArgs(args));
    //每隔 10s 重启一次,如果两分钟内重启过三次则停止 Job
    env.setRestartStrategy(RestartStrategies.failureRateRestart(3, Time.minutes(2), Time.seconds(10)));

    env.addSource(new SourceFunction<Long>() {
        @Override
        public void run(SourceContext<Long> sourceContext) throws Exception {
            while (true) {
                sourceContext.collect(null);
            }
        }
        @Override
        public void cancel() {
        }
    })
            .map((MapFunction<Long, Long>) aLong -> aLong / 1)
            .print();

    env.execute("zhisheng failureRate Restart Strategy example");
}
 
Example #9
Source File: KafkaExampleUtil.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static StreamExecutionEnvironment prepareExecutionEnv(ParameterTool parameterTool)
	throws Exception {

	if (parameterTool.getNumberOfParameters() < 5) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--zookeeper.connect <zk quorum> --group.id <some id>");
		throw new Exception("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--zookeeper.connect <zk quorum> --group.id <some id>");
	}

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();
	env.getConfig().setRestartStrategy(RestartStrategies.fixedDelayRestart(4, 10000));
	env.enableCheckpointing(5000); // create a checkpoint every 5 seconds
	env.getConfig().setGlobalJobParameters(parameterTool); // make parameters available in the web interface
	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

	return env;
}
 
Example #10
Source File: DataSetBrocastMain.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

    //1. 待广播的数据
    DataSet<Integer> toBroadcast = env.fromElements(1, 2, 3);

    env.fromElements("a", "b")
            .map(new RichMapFunction<String, String>() {
                List<Integer> broadcastData;

                @Override
                public void open(Configuration parameters) throws Exception {
                    // 3. 获取广播的DataSet数据 作为一个Collection
                    broadcastData = getRuntimeContext().getBroadcastVariable("zhisheng");
                }

                @Override
                public String map(String value) throws Exception {
                    return broadcastData.get(1) + value;
                }
            }).withBroadcastSet(toBroadcast, "zhisheng")// 2. 广播DataSet
            .print();
}
 
Example #11
Source File: DoubleParameter.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(ParameterTool parameterTool) {
	value = hasDefaultValue ? parameterTool.getDouble(name, defaultValue) : parameterTool.getDouble(name);

	if (hasMinimumValue) {
		if (minimumValueInclusive) {
			Util.checkParameter(value >= minimumValue,
				name + " must be greater than or equal to " + minimumValue);
		} else {
			Util.checkParameter(value > minimumValue,
				name + " must be greater than " + minimumValue);
		}
	}

	if (hasMaximumValue) {
		if (maximumValueInclusive) {
			Util.checkParameter(value <= maximumValue,
				name + " must be less than or equal to " + maximumValue);
		} else {
			Util.checkParameter(value < maximumValue,
				name + " must be less than " + maximumValue);
		}
	}
}
 
Example #12
Source File: DoubleParameter.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(ParameterTool parameterTool) {
	value = hasDefaultValue ? parameterTool.getDouble(name, defaultValue) : parameterTool.getDouble(name);

	if (hasMinimumValue) {
		if (minimumValueInclusive) {
			Util.checkParameter(value >= minimumValue,
				name + " must be greater than or equal to " + minimumValue);
		} else {
			Util.checkParameter(value > minimumValue,
				name + " must be greater than " + minimumValue);
		}
	}

	if (hasMaximumValue) {
		if (maximumValueInclusive) {
			Util.checkParameter(value <= maximumValue,
				name + " must be less than or equal to " + maximumValue);
		} else {
			Util.checkParameter(value < maximumValue,
				name + " must be less than " + maximumValue);
		}
	}
}
 
Example #13
Source File: GridGraph.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(ParameterTool parameterTool) throws ProgramParametrizationException {
	super.configure(parameterTool);

	// add dimensions as ordered by dimension ID (dim0, dim1, dim2, ...)

	Map<Integer, String> dimensionMap = new TreeMap<>();

	// first parse all dimensions into a sorted map
	for (String key : parameterTool.toMap().keySet()) {
		if (key.startsWith(PREFIX)) {
			int dimensionId = Integer.parseInt(key.substring(PREFIX.length()));
			dimensionMap.put(dimensionId, parameterTool.get(key));
		}
	}

	// then store dimensions in order
	for (String field : dimensionMap.values()) {
		dimensions.add(new Dimension(field));
	}
}
 
Example #14
Source File: Sink2ES6Main.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);
    DataStreamSource<MetricEvent> data = KafkaConfigUtil.buildSource(env);

    List<HttpHost> esAddresses = ESSinkUtil.getEsAddresses(parameterTool.get(ELASTICSEARCH_HOSTS));
    int bulkSize = parameterTool.getInt(ELASTICSEARCH_BULK_FLUSH_MAX_ACTIONS, 40);
    int sinkParallelism = parameterTool.getInt(STREAM_SINK_PARALLELISM, 5);

    log.info("-----esAddresses = {}, parameterTool = {}, ", esAddresses, parameterTool);

    ESSinkUtil.addSink(esAddresses, bulkSize, sinkParallelism, data,
            (MetricEvent metric, RuntimeContext runtimeContext, RequestIndexer requestIndexer) -> {
                requestIndexer.add(Requests.indexRequest()
                        .index(ZHISHENG + "_" + metric.getName())
                        .type(ZHISHENG)
                        .source(GsonUtil.toJSONBytes(metric), XContentType.JSON));
            },
            parameterTool);
    env.execute("flink learning connectors es6");
}
 
Example #15
Source File: FlinkKafkaSchemaTest1.java    From flink-learning with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
    final ParameterTool parameterTool = ExecutionEnvUtil.createParameterTool(args);
    StreamExecutionEnvironment env = ExecutionEnvUtil.prepare(parameterTool);
    Properties props = buildKafkaProps(parameterTool);
    //kafka topic list
    List<String> topics = Arrays.asList(parameterTool.get("metrics.topic"));
    FlinkKafkaConsumer011<MetricEvent> consumer = new FlinkKafkaConsumer011<>(topics, new KafkaDeserializationSchemaWrapper<>(new MetricSchema()), props);

    DataStreamSource<MetricEvent> data = env.addSource(consumer);

    data.print();

    env.execute("flink kafka connector test");
}
 
Example #16
Source File: IterationConvergence.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public void configure(ParameterTool parameterTool) {
	if (!parameterTool.has("iterations") && !parameterTool.has("convergence_threshold")) {
		// no configuration so use default iterations and maximum threshold
		value.iterations = defaultIterations;
		value.convergenceThreshold = Double.MAX_VALUE;
	} else {
		// use configured values and maximum default for unset values
		value.iterations = parameterTool.getInt("iterations", Integer.MAX_VALUE);
		Util.checkParameter(value.iterations > 0,
			"iterations must be greater than zero");

		value.convergenceThreshold = parameterTool.getDouble("convergence_threshold", Double.MAX_VALUE);
		Util.checkParameter(value.convergenceThreshold > 0,
			"convergence threshold must be greater than zero");
	}
}
 
Example #17
Source File: KafkaExample.java    From flink with Apache License 2.0 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);
	StreamExecutionEnvironment env = KafkaExampleUtil.prepareExecutionEnv(parameterTool);

	DataStream<KafkaEvent> input = env
		.addSource(
			new FlinkKafkaConsumer<>(
				parameterTool.getRequired("input-topic"),
				new KafkaEventSchema(),
				parameterTool.getProperties())
				.assignTimestampsAndWatermarks(new CustomWatermarkExtractor()))
		.keyBy("word")
		.map(new RollingAdditionMapper());

	input.addSink(
		new FlinkKafkaProducer<>(
			parameterTool.getRequired("output-topic"),
			new KeyedSerializationSchemaWrapper<>(new KafkaEventSchema()),
			parameterTool.getProperties(),
			FlinkKafkaProducer.Semantic.EXACTLY_ONCE));

	env.execute("Modern Kafka Example");
}
 
Example #18
Source File: DoubleParameterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithoutDefaultWithoutParameter() {
	Assert.assertEquals("--test TEST", parameter.getUsage());

	expectedException.expect(RuntimeException.class);
	expectedException.expectMessage("No data for required key 'test'");

	parameter.configure(ParameterTool.fromArgs(new String[]{}));
}
 
Example #19
Source File: TtlTestConfig.java    From flink with Apache License 2.0 5 votes vote down vote up
static TtlTestConfig fromArgs(ParameterTool pt) {
	int keySpace = pt.getInt(UPDATE_GENERATOR_SRC_KEYSPACE.key(), UPDATE_GENERATOR_SRC_KEYSPACE.defaultValue());
	long sleepAfterElements = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.key(),
		UPDATE_GENERATOR_SRC_SLEEP_AFTER_ELEMENTS.defaultValue());
	long sleepTime = pt.getLong(UPDATE_GENERATOR_SRC_SLEEP_TIME.key(),
		UPDATE_GENERATOR_SRC_SLEEP_TIME.defaultValue());
	Time ttl = Time.milliseconds(pt.getLong(STATE_TTL_VERIFIER_TTL_MILLI.key(),
		STATE_TTL_VERIFIER_TTL_MILLI.defaultValue()));
	long reportStatAfterUpdatesNum = pt.getLong(REPORT_STAT_AFTER_UPDATES_NUM.key(),
		REPORT_STAT_AFTER_UPDATES_NUM.defaultValue());
	return new TtlTestConfig(keySpace, sleepAfterElements, sleepTime, ttl, reportStatAfterUpdatesNum);
}
 
Example #20
Source File: AlertRuleAsyncIOFunction.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    parameterTool = (ParameterTool) getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
    connection = getConnection();
    String sql = "select * from alert_rule where name = ?;";
    if (connection != null) {
        ps = this.connection.prepareStatement(sql);
    }
}
 
Example #21
Source File: KafkaConfigUtil.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
private static Map<KafkaTopicPartition, Long> buildOffsetByTime(Properties props, ParameterTool parameterTool, Long time) {
    props.setProperty("group.id", "query_time_" + time);
    KafkaConsumer consumer = new KafkaConsumer(props);
    List<PartitionInfo> partitionsFor = consumer.partitionsFor(parameterTool.getRequired(PropertiesConstants.METRICS_TOPIC));
    Map<TopicPartition, Long> partitionInfoLongMap = new HashMap<>();
    for (PartitionInfo partitionInfo : partitionsFor) {
        partitionInfoLongMap.put(new TopicPartition(partitionInfo.topic(), partitionInfo.partition()), time);
    }
    Map<TopicPartition, OffsetAndTimestamp> offsetResult = consumer.offsetsForTimes(partitionInfoLongMap);
    Map<KafkaTopicPartition, Long> partitionOffset = new HashMap<>();
    offsetResult.forEach((key, value) -> partitionOffset.put(new KafkaTopicPartition(key.topic(), key.partition()), value.offset()));

    consumer.close();
    return partitionOffset;
}
 
Example #22
Source File: DoubleParameterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testMinAndMaxAtRangeMaximumInclusive() {
	parameter.setMinimumValue(-1, true);
	parameter.setMaximumValue(1, true);
	parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "1"}));
	Assert.assertEquals(new Double(1), parameter.getValue());
}
 
Example #23
Source File: ChoiceParameterTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithDefaultWithoutParameter() {
	parameter.setDefaultValue("default").addChoices("c0", "c1", "c2");
	Assert.assertEquals("[--choice <default | c0 | c1 | c2>]", parameter.getUsage());

	parameter.configure(ParameterTool.fromArgs(new String[]{}));
	Assert.assertEquals("default", parameter.getValue());
}
 
Example #24
Source File: Elasticsearch5SinkExample.java    From flink with Apache License 2.0 5 votes vote down vote up
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) {
	Map<String, Object> json = new HashMap<>();
	json.put("data", element);

	return Requests.indexRequest()
		.index(parameterTool.getRequired("index"))
		.type(parameterTool.getRequired("type"))
		.id(element)
		.source(json);
}
 
Example #25
Source File: DoubleParameterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithDefaultWithParameter() {
	parameter.setDefaultValue(43.21);
	Assert.assertEquals("[--test TEST]", parameter.getUsage());

	parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "12.34"}));
	Assert.assertEquals(new Double(12.34), parameter.getValue());
}
 
Example #26
Source File: GetAlertRuleSourceFunction.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
@Override
public void open(Configuration parameters) throws Exception {
    parameterTool = (ParameterTool) getRuntimeContext().getExecutionConfig().getGlobalJobParameters();
    connection = getConnection();
    String sql = "select * from alert_rule;";
    if (connection != null) {
        ps = this.connection.prepareStatement(sql);
    }
}
 
Example #27
Source File: Main.java    From flink-learning with Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
        final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
        ParameterTool parameterTool = ExecutionEnvUtil.PARAMETER_TOOL;
        Properties props = KafkaConfigUtil.buildKafkaProps(parameterTool);

        SingleOutputStreamOperator<Tuple2<String, String>> product = env.addSource(new FlinkKafkaConsumer011<>(
                parameterTool.get(METRICS_TOPIC),   //这个 kafka topic 需要和上面的工具类的 topic 一致
                new SimpleStringSchema(),
                props))
                .map(string -> GsonUtil.fromJson(string, ProductEvent.class)) //反序列化 JSON
                .flatMap(new FlatMapFunction<ProductEvent, Tuple2<String, String>>() {
                    @Override
                    public void flatMap(ProductEvent value, Collector<Tuple2<String, String>> out) throws Exception {
                        //收集商品 id 和 price 两个属性
                        out.collect(new Tuple2<>(value.getId().toString(), value.getPrice().toString()));
                    }
                });
//        product.print();

        //单个 Redis
        FlinkJedisPoolConfig conf = new FlinkJedisPoolConfig.Builder().setHost(parameterTool.get("redis.host")).build();
        product.addSink(new RedisSink<Tuple2<String, String>>(conf, new RedisSinkMapper()));

        //Redis 的 ip 信息一般都从配置文件取出来
        //Redis 集群
/*        FlinkJedisClusterConfig clusterConfig = new FlinkJedisClusterConfig.Builder()
                .setNodes(new HashSet<InetSocketAddress>(
                        Arrays.asList(new InetSocketAddress("redis1", 6379)))).build();*/

        //Redis Sentinels
/*        FlinkJedisSentinelConfig sentinelConfig = new FlinkJedisSentinelConfig.Builder()
                .setMasterName("master")
                .setSentinels(new HashSet<>(Arrays.asList("sentinel1", "sentinel2")))
                .setPassword("")
                .setDatabase(1).build();*/

        env.execute("flink redis connector");
    }
 
Example #28
Source File: ChoiceParameterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithDefaultWithParameter() {
	parameter.setDefaultValue("default").addChoices("c0", "c1", "c2");
	Assert.assertEquals("[--choice <default | c0 | c1 | c2>]", parameter.getUsage());

	parameter.configure(ParameterTool.fromArgs(new String[]{"--choice", "c1"}));
	Assert.assertEquals("c1", parameter.getValue());
}
 
Example #29
Source File: LongParameterTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testWithoutDefaultWithParameter() {
	Assert.assertEquals("--test TEST", parameter.getUsage());

	parameter.configure(ParameterTool.fromArgs(new String[]{"--test", "42"}));
	Assert.assertEquals(new Long(42), parameter.getValue());
}
 
Example #30
Source File: DataStreamAllroundTestJobFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void setupCheckpointing(final StreamExecutionEnvironment env, final ParameterTool pt) {
	String semantics = pt.get(TEST_SEMANTICS.key(), TEST_SEMANTICS.defaultValue());
	long checkpointInterval = pt.getLong(ENVIRONMENT_CHECKPOINT_INTERVAL.key(), ENVIRONMENT_CHECKPOINT_INTERVAL.defaultValue());
	CheckpointingMode checkpointingMode = semantics.equalsIgnoreCase("exactly-once")
		? CheckpointingMode.EXACTLY_ONCE
		: CheckpointingMode.AT_LEAST_ONCE;

	env.enableCheckpointing(checkpointInterval, checkpointingMode);

	boolean enableExternalizedCheckpoints = pt.getBoolean(
		ENVIRONMENT_EXTERNALIZE_CHECKPOINT.key(),
		ENVIRONMENT_EXTERNALIZE_CHECKPOINT.defaultValue());

	if (enableExternalizedCheckpoints) {
		String cleanupModeConfig = pt.get(
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.key(),
			ENVIRONMENT_EXTERNALIZE_CHECKPOINT_CLEANUP.defaultValue());

		CheckpointConfig.ExternalizedCheckpointCleanup cleanupMode;
		switch (cleanupModeConfig) {
			case "retain":
				cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.RETAIN_ON_CANCELLATION;
				break;
			case "delete":
				cleanupMode = CheckpointConfig.ExternalizedCheckpointCleanup.DELETE_ON_CANCELLATION;
				break;
			default:
				throw new IllegalArgumentException("Unknown clean up mode for externalized checkpoints: " + cleanupModeConfig);
		}
		env.getCheckpointConfig().enableExternalizedCheckpoints(cleanupMode);

		final int tolerableDeclinedCheckpointNumber = pt.getInt(
			ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.key(),
			ENVIRONMENT_TOLERABLE_DECLINED_CHECKPOINT_NUMBER.defaultValue());
		env.getCheckpointConfig().setTolerableCheckpointFailureNumber(tolerableDeclinedCheckpointNumber);
	}
}