Java Code Examples for org.apache.flink.streaming.api.datastream.DataStreamSource#map()

The following examples show how to use org.apache.flink.streaming.api.datastream.DataStreamSource#map() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: 163-bigdate-note   File: JavaCustomSinkToMySQL.java    License: GNU General Public License v3.0 6 votes vote down vote up 
public static void main(String[] args) throws Exception {
    StreamExecutionEnvironment environment = StreamExecutionEnvironment.getExecutionEnvironment();

    DataStreamSource<String> source = environment.socketTextStream("localhost", 7777);
    SingleOutputStreamOperator<Student> studentString = source.map(new MapFunction<String, Student>() {
        @Override
        public Student map(String value) throws Exception {
            Student student = new Student();
            String[] strings = value.split(",");
            student.setId(Integer.parseInt(strings[0]));
            student.setName(strings[1]);
            student.setAge(Integer.parseInt(strings[2]));
            return student;
        }
    });

    studentString.addSink(new SinkToMysql());

    environment.execute("JavaCustomSinkToMySQL");
}
Example 2
Source Project: Flink-CEPplus   File: TestJob.java    License: Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
Example 3
Source Project: flink   File: TestJob.java    License: Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
Example 4
Source Project: Alink   File: NumSeqSourceStreamOp.java    License: Apache License 2.0 5 votes vote down vote up 
public NumSeqSourceStreamOp(long from, long to, String colName, double timePerSample, Params params) {
    super(params);

    DataStreamSource<Long> seq = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamExecutionEnvironment().generateSequence(from, to);
    DataStream<Long> data = seq.map(new transform(new Double[]{timePerSample}));

    this.setOutputTable(MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment().fromDataStream(data, colName));
}
Example 5
Source Project: Alink   File: NumSeqSourceStreamOp.java    License: Apache License 2.0 5 votes vote down vote up 
public NumSeqSourceStreamOp(long from, long to, String colName, Double[] timeZones, Params params) {
    super(params);

    DataStreamSource<Long> seq = MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamExecutionEnvironment().generateSequence(from, to);
    DataStream<Long> data = seq.map(new transform(timeZones));

    this.setOutputTable(MLEnvironmentFactory.get(getMLEnvironmentId()).getStreamTableEnvironment().fromDataStream(data, colName));
}
Example 6
Source Project: flink   File: TestJob.java    License: Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
Example 7
Source Project: flink   File: TestUserClassLoaderJob.java    License: Apache License 2.0 5 votes vote down vote up 
public static void main(String[] args) throws Exception {
	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	final DataStreamSource<Integer> source = env.fromElements(new TestUserClassLoaderJobLib().getValue(), 1, 2, 3, 4);
	final SingleOutputStreamOperator<Integer> mapper = source.map(element -> 2 * element);
	mapper.addSink(new DiscardingSink<>());

	ParameterTool parameterTool = ParameterTool.fromArgs(args);
	env.execute(TestUserClassLoaderJob.class.getCanonicalName() + "-" + parameterTool.getRequired("arg"));
}
Example 8
Source Project: Flink-CEPplus   File: TestAvroConsumerConfluent.java    License: Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);

	if (parameterTool.getNumberOfParameters() < 6) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--zookeeper.connect <zk quorum> " +
			"--schema-registry-url <confluent schema registry> --group.id <some id>");
		return;
	}
	Properties config = new Properties();
	config.setProperty("bootstrap.servers", parameterTool.getRequired("bootstrap.servers"));
	config.setProperty("group.id", parameterTool.getRequired("group.id"));
	config.setProperty("zookeeper.connect", parameterTool.getRequired("zookeeper.connect"));
	String schemaRegistryUrl = parameterTool.getRequired("schema-registry-url");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataStreamSource<User> input = env
		.addSource(
			new FlinkKafkaConsumer010<>(
				parameterTool.getRequired("input-topic"),
				ConfluentRegistryAvroDeserializationSchema.forSpecific(User.class, schemaRegistryUrl),
				config).setStartFromEarliest());

	SingleOutputStreamOperator<String> mapToString = input
		.map((MapFunction<User, String>) SpecificRecordBase::toString);

	FlinkKafkaProducer010<String> stringFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
		parameterTool.getRequired("output-topic"),
		new SimpleStringSchema(),
		config);

	mapToString.addSink(stringFlinkKafkaProducer010);
	env.execute("Kafka 0.10 Confluent Schema Registry AVRO Example");
}
Example 9
Source Project: flink   File: TestAvroConsumerConfluent.java    License: Apache License 2.0 4 votes vote down vote up 
public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);

	if (parameterTool.getNumberOfParameters() < 6) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--zookeeper.connect <zk quorum> " +
			"--schema-registry-url <confluent schema registry> --group.id <some id>");
		return;
	}
	Properties config = new Properties();
	config.setProperty("bootstrap.servers", parameterTool.getRequired("bootstrap.servers"));
	config.setProperty("group.id", parameterTool.getRequired("group.id"));
	config.setProperty("zookeeper.connect", parameterTool.getRequired("zookeeper.connect"));
	String schemaRegistryUrl = parameterTool.getRequired("schema-registry-url");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().disableSysoutLogging();

	DataStreamSource<User> input = env
		.addSource(
			new FlinkKafkaConsumer010<>(
				parameterTool.getRequired("input-topic"),
				ConfluentRegistryAvroDeserializationSchema.forSpecific(User.class, schemaRegistryUrl),
				config).setStartFromEarliest());

	SingleOutputStreamOperator<String> mapToString = input
		.map((MapFunction<User, String>) SpecificRecordBase::toString);

	FlinkKafkaProducer010<String> stringFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
		parameterTool.getRequired("output-topic"),
		new SimpleStringSchema(),
		config);

	mapToString.addSink(stringFlinkKafkaProducer010);
	env.execute("Kafka 0.10 Confluent Schema Registry AVRO Example");
}
Example 10
Source Project: flink   File: TestAvroConsumerConfluent.java    License: Apache License 2.0 3 votes vote down vote up 
public static void main(String[] args) throws Exception {
	// parse input arguments
	final ParameterTool parameterTool = ParameterTool.fromArgs(args);

	if (parameterTool.getNumberOfParameters() < 6) {
		System.out.println("Missing parameters!\n" +
			"Usage: Kafka --input-topic <topic> --output-string-topic <topic> --output-avro-topic <topic> " +
			"--bootstrap.servers <kafka brokers> " +
			"--schema-registry-url <confluent schema registry> --group.id <some id>");
		return;
	}
	Properties config = new Properties();
	config.setProperty("bootstrap.servers", parameterTool.getRequired("bootstrap.servers"));
	config.setProperty("group.id", parameterTool.getRequired("group.id"));
	String schemaRegistryUrl = parameterTool.getRequired("schema-registry-url");

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	DataStreamSource<User> input = env
		.addSource(
			new FlinkKafkaConsumer010<>(
				parameterTool.getRequired("input-topic"),
				ConfluentRegistryAvroDeserializationSchema.forSpecific(User.class, schemaRegistryUrl),
				config).setStartFromEarliest());

	SingleOutputStreamOperator<String> mapToString = input
		.map((MapFunction<User, String>) SpecificRecordBase::toString);

	FlinkKafkaProducer010<String> stringFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
		parameterTool.getRequired("output-string-topic"),
		new SimpleStringSchema(),
		config);
	mapToString.addSink(stringFlinkKafkaProducer010);

	FlinkKafkaProducer010<User> avroFlinkKafkaProducer010 = new FlinkKafkaProducer010<>(
			parameterTool.getRequired("output-avro-topic"),
			ConfluentRegistryAvroSerializationSchema.forSpecific(User.class, parameterTool.getRequired("output-subject"), schemaRegistryUrl),
			config);
	input.addSink(avroFlinkKafkaProducer010);

	env.execute("Kafka 0.10 Confluent Schema Registry AVRO Example");
}