Java Code Examples for org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat

The following examples show how to use org.apache.flink.api.java.hadoop.mapred.HadoopInputFormat. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
 
Example 2
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
 
Example 3
Source Project: Flink-CEPplus   Source File: HadoopMapredCompatWordCount.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
 
Example 4
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
 
Example 5
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
 
Example 6
Source Project: flink   Source File: HadoopMapredCompatWordCount.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
 
Example 7
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath, JobConf job) {
	// set input path in JobConf
	org.apache.hadoop.mapred.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapredInputFormat, key, value, job);
}
 
Example 8
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath, Job job) throws IOException {
	// set input path in Job
	org.apache.hadoop.mapreduce.lib.input.FileInputFormat.addInputPath(job, new org.apache.hadoop.fs.Path(inputPath));
	// return wrapping InputFormat
	return createHadoopInput(mapreduceInputFormat, key, value, job);
}
 
Example 9
Source Project: flink   Source File: HadoopMapredCompatWordCount.java    License: Apache License 2.0 5 votes vote down vote up
public static void main(String[] args) throws Exception {
	if (args.length < 2) {
		System.err.println("Usage: WordCount <input path> <result path>");
		return;
	}

	final String inputPath = args[0];
	final String outputPath = args[1];

	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();

	// Set up the Hadoop Input Format
	HadoopInputFormat<LongWritable, Text> hadoopInputFormat = new HadoopInputFormat<LongWritable, Text>(new TextInputFormat(), LongWritable.class, Text.class, new JobConf());
	TextInputFormat.addInputPath(hadoopInputFormat.getJobConf(), new Path(inputPath));

	// Create a Flink job with it
	DataSet<Tuple2<LongWritable, Text>> text = env.createInput(hadoopInputFormat);

	DataSet<Tuple2<Text, LongWritable>> words =
			text.flatMap(new HadoopMapFunction<LongWritable, Text, Text, LongWritable>(new Tokenizer()))
				.groupBy(0).reduceGroup(new HadoopReduceCombineFunction<Text, LongWritable, Text, LongWritable>(new Counter(), new Counter()));

	// Set up Hadoop Output Format
	HadoopOutputFormat<Text, LongWritable> hadoopOutputFormat =
			new HadoopOutputFormat<Text, LongWritable>(new TextOutputFormat<Text, LongWritable>(), new JobConf());
	hadoopOutputFormat.getJobConf().set("mapred.textoutputformat.separator", " ");
	TextOutputFormat.setOutputPath(hadoopOutputFormat.getJobConf(), new Path(outputPath));

	// Output & Execute
	words.output(hadoopOutputFormat).setParallelism(1);
	env.execute("Hadoop Compat WordCount");
}
 
Example 10
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
 
Example 11
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
 
Example 12
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
 
Example 13
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
 
Example 14
Source Project: Flink-CEPplus   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}
 
Example 15
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
 
Example 16
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
 
Example 17
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
 
Example 18
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
 
Example 19
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}
 
Example 20
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readHadoopFile(org.apache.hadoop.mapred.FileInputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, String inputPath) {
	return readHadoopFile(mapredInputFormat, key, value, inputPath, new JobConf());
}
 
Example 21
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} to read a Hadoop sequence file for the given key and value classes.
 *
 * @return A Flink InputFormat that wraps a Hadoop SequenceFileInputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> readSequenceFile(Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(new org.apache.hadoop.mapred.SequenceFileInputFormat<K, V>(), key, value, inputPath);
}
 
Example 22
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapred.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> HadoopInputFormat<K, V> createHadoopInput(org.apache.hadoop.mapred.InputFormat<K, V> mapredInputFormat, Class<K> key, Class<V> value, JobConf job) {
	return new HadoopInputFormat<>(mapredInputFormat, key, value, job);
}
 
Example 23
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.lib.input.FileInputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop FileInputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> readHadoopFile(
		org.apache.hadoop.mapreduce.lib.input.FileInputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, String inputPath) throws IOException {
	return readHadoopFile(mapreduceInputFormat, key, value, inputPath, Job.getInstance());
}
 
Example 24
Source Project: flink   Source File: HadoopInputs.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Creates a Flink {@link InputFormat} that wraps the given Hadoop {@link org.apache.hadoop.mapreduce.InputFormat}.
 *
 * @return A Flink InputFormat that wraps the Hadoop InputFormat.
 */
public static <K, V> org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<K, V> createHadoopInput(
		org.apache.hadoop.mapreduce.InputFormat<K, V> mapreduceInputFormat, Class<K> key, Class<V> value, Job job) {
	return new org.apache.flink.api.java.hadoop.mapreduce.HadoopInputFormat<>(mapreduceInputFormat, key, value, job);
}